Print failed test results at the end for the spec test suite (#8154)

* Print all failed test stdouts/stderrs at the end so that the output
isn't lost among the successful tests
* Make `shared.options.abort_on_first_failure` work with the spec test
suite and add aliases for --fail-fast and --no-fail-fast
* Re-raise in run_one_spec_test since fail_with_error may no longer
throw an exception when abort_on_first_failure is false (the remaining
code shouldn't execute because then `actual` isn't defined).

Author: stevenfontanella

check.py

@@ -21,6 +21,7 @@
 import sys
 import unittest
 from collections import OrderedDict
+from contextlib import contextmanager
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
 
@@ -219,6 +220,7 @@ def run_one_spec_test(wast: Path, stdout=None):
             return  # don't try all the binary format stuff TODO
         else:
             shared.fail_with_error(str(e))
+            raise
 
     check_expected(actual, expected, stdout=stdout)
 
@@ -248,32 +250,64 @@ def run_one_spec_test(wast: Path, stdout=None):
 
 
 def run_spec_test_with_wrapped_stdout(wast: Path):
+    """Return (bool, str) where the first element is whether the test was
+    successful and the second is the combined stdout and stderr of the test.
+    """
     out = io.StringIO()
     try:
         run_one_spec_test(wast, stdout=out)
     except Exception as e:
         # Serialize exceptions into the output string buffer
         # so they can be reported on the main thread.
         print(e, file=out)
-        raise
-    return out.getvalue()
+        return False, out.getvalue()
+    return True, out.getvalue()
+
+
+@contextmanager
+def red_output(file=sys.stdout):
+    print("\033[31m", end="", file=file)
+    try:
+        yield
+    finally:
+        print("\033[0m", end="", file=file)
+
+
+def red_stderr():
+    return red_output(file=sys.stderr)
 
 
 def run_spec_tests():
     print('\n[ checking wasm-shell spec testcases... ]\n')
 
     worker_count = os.cpu_count()
     print("Running with", worker_count, "workers")
-    test_paths = [Path(x) for x in shared.options.spec_tests]
+    test_paths = (Path(x) for x in shared.options.spec_tests)
+
+    failed_stdouts = []
     with ThreadPool(processes=worker_count) as pool:
         try:
-            for result in pool.imap_unordered(run_spec_test_with_wrapped_stdout, test_paths):
-                print(result, end="")
+            for success, stdout in pool.imap_unordered(run_spec_test_with_wrapped_stdout, test_paths):
+                if success:
+                    print(stdout, end="")
+                    continue
+
+                failed_stdouts.append(stdout)
+                if shared.options.abort_on_first_failure:
+                    with red_stderr():
+                        print("Aborted spec test suite execution after first failure. Set --no-fail-fast to disable this.", file=sys.stderr)
+                    break
         except KeyboardInterrupt:
             # Hard exit to avoid threads continuing to run after Ctrl-C.
             # There's no concern of deadlocking during shutdown here.
             os._exit(1)
 
+    if failed_stdouts:
+        with red_stderr():
+            print("Failed tests:", file=sys.stderr)
+            for failed in failed_stdouts:
+                print(failed, end="", file=sys.stderr)
+
 
 def run_validator_tests():
     print('\n[ running validation tests... ]\n')

scripts/test/shared.py

@@ -40,15 +40,10 @@ def parse_args(args):
         '--no-torture', dest='torture', action='store_false',
         help='Disables running the torture testcases.')
     parser.add_argument(
-        '--abort-on-first-failure', dest='abort_on_first_failure',
-        action='store_true', default=True,
+        '--abort-on-first-failure', '--fail-fast', dest='abort_on_first_failure',
+        action=argparse.BooleanOptionalAction, default=True,
         help=('Specifies whether to halt test suite execution on first test error.'
               ' Default: true.'))
-    parser.add_argument(
-        '--no-abort-on-first-failure', dest='abort_on_first_failure',
-        action='store_false',
-        help=('If set, the whole test suite will run to completion independent of'
-              ' earlier errors.'))
     parser.add_argument(
         '--binaryen-bin', dest='binaryen_bin', default='',
         help=('Specifies the path to the Binaryen executables in the CMake build'