From ca4d26dc6b69fe5b5ab90b7876f9dc3d34c8b2df Mon Sep 17 00:00:00 2001 From: Ben Wilfong <48168887+wilfonba@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:46:23 -0500 Subject: [PATCH 1/4] asdf --- toolchain/mfc/args.py | 1 + toolchain/mfc/state.py | 1 - toolchain/mfc/test/cases.py | 3 +++ toolchain/mfc/test/test.py | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index cad27120a0..bda830d549 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -81,6 +81,7 @@ def add_common_arguments(p, mask = None): test.add_argument("-%", "--percent", type=int, default=100, help="Percentage of tests to run.") test.add_argument("-m", "--max-attempts", type=int, default=1, help="Maximum number of attempts to run a test.") test.add_argument( "--no-build", action="store_true", default=False, help="(Testing) Do not rebuild MFC.") + test.add_argument( "--no-examples", action="store_true", default=False, help="Do not test example cases." ) test.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.") test_meg = test.add_mutually_exclusive_group() test_meg.add_argument("--generate", action="store_true", default=False, help="(Test Generation) Generate golden files.") diff --git a/toolchain/mfc/state.py b/toolchain/mfc/state.py index c3dcc8a3ba..7c21408a23 100644 --- a/toolchain/mfc/state.py +++ b/toolchain/mfc/state.py @@ -50,7 +50,6 @@ def __str__(self) -> str: gCFG: MFCConfig = MFCConfig() gARG: dict = {} - def ARG(arg: str, dflt = None) -> typing.Any: # pylint: disable=global-variable-not-assigned global gARG diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py index f6fea37f6e..b0e51525d6 100644 --- a/toolchain/mfc/test/cases.py +++ b/toolchain/mfc/test/cases.py @@ -1,6 +1,7 @@ import os, typing, itertools from mfc import common +from ..state import ARG from .case import Nt, define_case_d, define_case_f, CaseGeneratorStack, TestCaseBuilder def get_bc_mods(bc: int, dimInfo): @@ -861,7 +862,9 @@ def chemistry_cases(): )) foreach_dimension() + foreach_example() + chemistry_cases() # Sanity Check 1 diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index ac0d13d2e7..85ae568c91 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -56,6 +56,9 @@ def __filter(cases_) -> typing.List[TestCase]: cases.remove(case) skipped_cases.append(case) + if ARG("no_examples"): + cases = [case for case in cases if not ("Example" in case.trace)] + if ARG("percent") == 100: return cases, skipped_cases From e76c6964164f089de9e12ea49d151845fcb707eb Mon Sep 17 00:00:00 2001 From: Ben Wilfong <48168887+wilfonba@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:53:10 -0500 Subject: [PATCH 2/4] lint the toolchain --- toolchain/mfc/test/cases.py | 1 - toolchain/mfc/test/test.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py index b0e51525d6..8d78af9f2a 100644 --- a/toolchain/mfc/test/cases.py +++ b/toolchain/mfc/test/cases.py @@ -1,7 +1,6 @@ import os, typing, itertools from mfc import common -from ..state import ARG from .case import Nt, define_case_d, define_case_f, CaseGeneratorStack, TestCaseBuilder def get_bc_mods(bc: int, dimInfo): diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 85ae568c91..9dfaab3e8e 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -57,7 +57,7 @@ def __filter(cases_) -> typing.List[TestCase]: skipped_cases.append(case) if ARG("no_examples"): - cases = [case for case in cases if not ("Example" in case.trace)] + cases = [case for case in cases if not "Example" in case.trace] if ARG("percent") == 100: return cases, skipped_cases From a115c55bb4e4fc0a0041567969c43d0975eedece Mon Sep 17 00:00:00 2001 From: Ben Wilfong <48168887+wilfonba@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:51:31 -0500 Subject: [PATCH 3/4] documentation --- docs/documentation/testing.md | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/docs/documentation/testing.md b/docs/documentation/testing.md index 249865b4da..48db9fcacf 100644 --- a/docs/documentation/testing.md +++ b/docs/documentation/testing.md @@ -1,5 +1,5 @@ ## Testing - + To run MFC's test suite, run ```shell ./mfc.sh test -j @@ -8,15 +8,15 @@ To run MFC's test suite, run It will generate and run test cases, comparing their output to previous runs from versions of MFC considered accurate. *golden files*, stored in the `tests/` directory contain this data, aggregating `.dat` files generated when running MFC. A test is considered passing when our error tolerances are met in order to maintain a high level of stability and accuracy. -Run `./mfc.sh test -h` for a full list of accepted arguments. - -Most notably, you can consult the full list of tests by running -```shell -./mfc.sh test -l -``` +`./mfc.sh test` has the following unique options: +- `-l` outputs the full list of tests +- `--from` (`-f)` and `--to` (`t`) restrict testing to a range of contiguous slugs +- `--only` (`-o`) restricts testing to a non-contiguous rage of tests based on if their trace contains a certain feature +- `--test-all` (`a`) test post process and ensure the Silo database files are correct +- `--percent` (`%`) to specify a percentage of the test suite to select at random and test +- `--max-attempts` (`-m`) the maximum number of attempts to make on a test before considering it failed +- `--no-examples` skips the testing of cases in the examples folder -To restrict to a given range, use the `--from` (`-f`) and `--to` (`-t`) options. -To run a (non-contiguous) subset of tests, use the `--only` (`-o`) option instead. To specify a computer, pass the `-c` flag to `./mfc.sh run` like so: ```shell ./mfc.sh test -j -- -c @@ -27,15 +27,13 @@ The use of `--` in the above command passes options to the `./mfc.sh run` comman ### Creating Tests -To (re)generate *golden files*, append the `--generate` option: -```shell -./mfc.sh test --generate -j 8 -``` +Creating and updating test cases can be done with the following command line arguments: +- `--generate` to generate golden files for a new test case +- `--add-new-variables` to similar to `--generate`, but rather than generating a golden file from scratch, it generates a gold file with new variables for an updated test without changing the original golden file values. +- `--remove-old-tests` to remove the directories of tests that no longer exist It is recommended that a range be specified when generating golden files for new test cases, as described in the previous section, in an effort not to regenerate the golden files of existing test cases. -**Note:** If you output new variables and want to update the golden files to include these without modifying the original data, use the `--add-new-variables` option instead. - Adding a new test case can be done by modifying [cases.py](https://github.com/MFlowCode/MFC/tree/master/toolchain/mfc/test/cases.py). The function `list_cases` is responsible for generating the list of test cases. Loops and conditionals are used to vary parameters, whose defaults can be found in the `BASE_CFG` case object within [case.py](https://github.com/MFlowCode/MFC/tree/master/toolchain/mfc/test/case.py). @@ -44,7 +42,7 @@ The function operates on two variables: - `stack`: A stack that holds the variations to the default case parameters. By pushing and popping the stack inside loops and conditionals, it is easier to nest test case descriptions, as it holds the variations that are common to all future test cases within the same indentation level (in most scenarios). -- `cases`: A list that holds fully-formed `Case` objects, that will be returned at the end of the function. +- `cases`: A list that holds fully-formed `Case` objects, that will be returned at the end of the function. Internally a test case is described as: ```python @@ -93,7 +91,7 @@ Finally, the case is appended to the `cases` list, which will be returned by the ### Testing Post Process -To test the post-processing code, append the `-a` or `--test-all` option: +To test the post-processing code, append the `-a` or `--test-all` option: ```shell ./mfc.sh test -a -j 8 ``` @@ -101,5 +99,5 @@ To test the post-processing code, append the `-a` or `--test-all` option: This argument will re-run the test stack with `parallel_io='T'`, which generates silo_hdf5 files. It will also turn most write parameters (`*_wrt`) on. Then, it searches through the silo files using `h5dump` to ensure that there are no `NaN`s or `Infinity`s. -Although adding this option does not guarantee that accurate `.silo` files are generated, it does ensure that the post-process code does not fail or produce malformed data. +Although adding this option does not guarantee that accurate `.silo` files are generated, it does ensure that the post-process code does not fail or produce malformed data. From 93c6297dce30478653ca7edfacf3dee4df5e98a4 Mon Sep 17 00:00:00 2001 From: Ben Wilfong <48168887+wilfonba@users.noreply.github.com> Date: Wed, 11 Dec 2024 23:35:28 -0500 Subject: [PATCH 4/4] spelling --- docs/documentation/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/documentation/testing.md b/docs/documentation/testing.md index 48db9fcacf..15b1ca3618 100644 --- a/docs/documentation/testing.md +++ b/docs/documentation/testing.md @@ -11,7 +11,7 @@ A test is considered passing when our error tolerances are met in order to maint `./mfc.sh test` has the following unique options: - `-l` outputs the full list of tests - `--from` (`-f)` and `--to` (`t`) restrict testing to a range of contiguous slugs -- `--only` (`-o`) restricts testing to a non-contiguous rage of tests based on if their trace contains a certain feature +- `--only` (`-o`) restricts testing to a non-contiguous range of tests based on if their trace contains a certain feature - `--test-all` (`a`) test post process and ensure the Silo database files are correct - `--percent` (`%`) to specify a percentage of the test suite to select at random and test - `--max-attempts` (`-m`) the maximum number of attempts to make on a test before considering it failed