diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index ff8ab585e..802259a6a 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -2,7 +2,7 @@ from colorama import Fore, init from . import TestReporter -from ..test_case import TestCase +from ..test_case import TestCase, Skipped from ..test_suite import TestSuite from ..runtime_adapter import RuntimeVersion @@ -19,7 +19,9 @@ def __init__(self, colored: bool = True) -> None: self._colored = colored def report_test(self, test: TestCase) -> None: - if test.result.failed: + if isinstance(test.result, Skipped): + self._print_skip(f"Test {test.name} skipped: {test.result.reason}") + elif test.result.failed: self._print_fail(f"Test {test.name} failed") for reason in test.result.failures: self._print_fail(f" [{reason.type}] {reason.message}") @@ -27,10 +29,8 @@ def report_test(self, test: TestCase) -> None: print(test.result.output.stdout) print("STDERR:") print(test.result.output.stderr) - elif test.result.is_executed: - self._print_pass(f"Test {test.name} passed") else: - self._print_skip(f"Test {test.name} skipped") + self._print_pass(f"Test {test.name} passed") def report_test_suite(self, test_suite: TestSuite) -> None: self._test_suites.append(test_suite) diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eebb95d3f..79b960848 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -1,6 +1,6 @@ import logging import json -from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional +from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional, Union class Output(NamedTuple): @@ -14,9 +14,8 @@ class Failure(NamedTuple): message: str -class Result(NamedTuple): +class ExecutedResult(NamedTuple): output: Output - is_executed: bool failures: List[Failure] @property @@ -24,6 +23,13 @@ def failed(self) -> bool: return len(self.failures) > 0 +class SkippedResult(NamedTuple): + reason: str + + +Result = Union[ExecutedResult, SkippedResult] + + T = TypeVar("T", bound="Config") diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py index 4681cd9a6..aab8f3094 100644 --- a/test-runner/wasi_test_runner/test_suite.py +++ b/test-runner/wasi_test_runner/test_suite.py @@ -1,6 +1,6 @@ from typing import NamedTuple, List from datetime import datetime -from .test_case import TestCase +from .test_case import TestCase, ExecutedResult, SkippedResult class TestSuite(NamedTuple): @@ -19,7 +19,7 @@ def pass_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed is False + if isinstance(test.result, ExecutedResult) and not test.result.failed ] ) @@ -29,10 +29,10 @@ def fail_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed + if isinstance(test.result, ExecutedResult) and test.result.failed ] ) @property def skip_count(self) -> int: - return len([1 for test in self.test_cases if not test.result.is_executed]) + return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)]) diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index c9a26b149..7f00aa342 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -11,7 +11,8 @@ from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( - Result, + ExecutedResult, + SkippedResult, Config, Output, TestCase, @@ -38,11 +39,13 @@ def run_tests_from_test_suite( for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")): test_name = os.path.splitext(os.path.basename(test_path))[0] for filt in filters: - # for now, just drop the skip reason string. it might be - # useful to make reporters report it. - skip, _ = filt.should_skip(test_suite_name, test_name) + skip, reason = filt.should_skip(test_suite_name, test_name) if skip: - test_case = _skip_single_test(runtime, validators, test_path) + # Given the type of should_skip, "if skip" implies reason + # is str. But mypy doesn't seem to recognize it. Use an + # explicit cast for now. + reason = cast(str, reason) + test_case = _skip_single_test(runtime, validators, test_path, reason) break else: test_case = _execute_single_test(runtime, validators, test_path) @@ -61,13 +64,13 @@ def run_tests_from_test_suite( def _skip_single_test( - _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str + _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str, reason: str ) -> TestCase: config = _read_test_config(test_path) return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=Result(output=Output(0, "", ""), is_executed=False, failures=[]), + result=SkippedResult(reason), duration_s=0, ) @@ -88,14 +91,14 @@ def _execute_single_test( ) -def _validate(validators: List[Validator], config: Config, output: Output) -> Result: +def _validate(validators: List[Validator], config: Config, output: Output) -> ExecutedResult: failures = [ result for result in [validator(config, output) for validator in validators] if result is not None ] - return Result(failures=failures, is_executed=True, output=output) + return ExecutedResult(failures=failures, output=output) def _read_test_config(test_path: str) -> Config: