Skip to content

Commit d2b5b3a

Browse files
yamtyamt
committed
add a dumb timeout logic
#42
1 parent a51037c commit d2b5b3a

File tree

6 files changed

+60
-29
lines changed

6 files changed

+60
-29
lines changed

‎test-runner/wasi_test_runner/reporters/console.py

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from colorama import Fore, init
33

44
from . import TestReporter
5-
from ..test_case import TestCase
5+
from ..test_case import TestCase, SkippedResult, TimedoutResult
66
from ..test_suite import TestSuite
77
from ..runtime_adapter import RuntimeVersion
88

@@ -20,18 +20,21 @@ def __init__(self, colored: bool = True) -> None:
2020
self._colored = colored
2121

2222
def report_test(self, test: TestCase) -> None:
23-
if test.result.failed:
24-
self._print_fail(f"Test {test.name} failed")
25-
for reason in test.result.failures:
26-
self._print_fail(f" [{reason.type}] {reason.message}")
27-
print("STDOUT:")
28-
print(test.result.output.stdout)
29-
print("STDERR:")
30-
print(test.result.output.stderr)
31-
elif test.result.is_executed:
32-
self._print_pass(f"Test {test.name} passed")
33-
else:
23+
if isinstance(test.result, TimedoutResult):
24+
self._print_fail(f"Test {test.name} timed out")
25+
elif isinstance(test.result, SkippedResult):
3426
self._print_skip(f"Test {test.name} skipped")
27+
else:
28+
if test.result.failed:
29+
self._print_fail(f"Test {test.name} failed")
30+
for reason in test.result.failures:
31+
self._print_fail(f" [{reason.type}] {reason.message}")
32+
print("STDOUT:")
33+
print(test.result.output.stdout)
34+
print("STDERR:")
35+
print(test.result.output.stderr)
36+
else:
37+
self._print_pass(f"Test {test.name} passed")
3538

3639
def report_test_suite(self, test_suite: TestSuite) -> None:
3740
self._test_suites.append(test_suite)
@@ -41,29 +44,31 @@ def finalize(self, version: RuntimeVersion) -> None:
4144
print("===== Test results =====")
4245
print(f"Runtime: {version.name} {version.version}")
4346

44-
total_skip = total_pass = total_fail = pass_suite = 0
47+
total_skip = total_pass = total_fail = total_timedout = pass_suite = 0
4548

4649
for suite in self._test_suites:
4750
total_pass += suite.pass_count
4851
total_fail += suite.fail_count
4952
total_skip += suite.skip_count
53+
total_timedout += suite.timedout_count
5054

51-
if suite.fail_count == 0:
55+
if suite.fail_count == 0 and suite.timedout_count == 0:
5256
pass_suite += 1
5357

5458
print(f"Suite: {suite.name}")
5559
print(f" Total: {suite.test_count}")
56-
self._print_pass(f" Passed: {suite.pass_count}")
57-
self._print_fail(f" Failed: {suite.fail_count}")
58-
self._print_skip(f" Skipped: {suite.skip_count}")
60+
self._print_pass(f" Passed: {suite.pass_count}")
61+
self._print_fail(f" Failed: {suite.fail_count}")
62+
self._print_skip(f" Skipped: {suite.skip_count}")
63+
self._print_fail(f" Timed out: {suite.timedout_count}")
5964
print("")
6065

6166
print(
62-
f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0)}"
67+
f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0, 0)}"
6368
)
64-
print(f"Tests: {self._get_summary(total_fail, total_pass, total_skip)}")
69+
print(f"Tests: {self._get_summary(total_fail, total_pass, total_skip, total_timedout)}")
6570

66-
def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str:
71+
def _get_summary(self, fail_count: int, pass_count: int, skip_count: int, timedout_count: int) -> str:
6772
items: List[str] = []
6873

6974
if fail_count:
@@ -72,6 +77,8 @@ def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str
7277
items.append(f"{self._pass_color}{pass_count} passed")
7378
if skip_count:
7479
items.append(f"{self._skip_color}{skip_count} skipped")
80+
if timedout_count:
81+
items.append(f"{self._fail_color}{timedout_count} timed out")
7582

7683
total = fail_count + pass_count + skip_count
7784
items.append(f"{self._reset_color}{total} total")

‎test-runner/wasi_test_runner/reporters/json.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
from . import TestReporter
77
from ..test_suite import TestSuite
8+
from ..test_case import Result, SkippedResult, TimedoutResult
89
from ..runtime_adapter import RuntimeVersion
910

1011

@@ -28,16 +29,19 @@ def finalize(self, version: RuntimeVersion) -> None:
2829
"duration_s": suite.duration_s,
2930
"failed": suite.fail_count,
3031
"skipped": suite.skip_count,
32+
"timedout": suite.timedout_count,
3133
"passed": suite.pass_count,
3234
"tests": [
3335
{
3436
"name": test.name,
35-
"executed": test.result.is_executed,
37+
"executed": isinstance(test.result, Result),
38+
"skipped": isinstance(test.result, SkippedResult),
39+
"timedout": isinstance(test.result, TimedoutResult),
3640
"duration_s": test.duration_s,
3741
"wasi_functions": test.config.wasi_functions,
3842
"failures": [
3943
failure.message for failure in test.result.failures
40-
],
44+
] if isinstance(test.result, Result) else [],
4145
}
4246
for test in suite.test_cases
4347
],

‎test-runner/wasi_test_runner/runtime_adapter.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ def run_test(
4848
text=True,
4949
check=False,
5050
cwd=Path(test_path).parent,
51+
timeout=3.
5152
)
5253
return Output(result.returncode, result.stdout, result.stderr)
5354

‎test-runner/wasi_test_runner/test_case.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ def failed(self) -> bool:
2424
return len(self.failures) > 0
2525

2626

27+
class TimedoutResult(NamedTuple):
28+
pass
29+
30+
31+
class SkippedResult(NamedTuple):
32+
pass
33+
34+
2735
T = TypeVar("T", bound="Config")
2836

2937

‎test-runner/wasi_test_runner/test_suite.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from typing import NamedTuple, List
22
from datetime import datetime
3-
from .test_case import TestCase
3+
from .test_case import TestCase, Result, SkippedResult, TimedoutResult
44

55

66
class TestSuite(NamedTuple):
@@ -19,7 +19,7 @@ def pass_count(self) -> int:
1919
[
2020
1
2121
for test in self.test_cases
22-
if test.result.is_executed and test.result.failed is False
22+
if isinstance(test.result, Result) and test.result.failed is False
2323
]
2424
)
2525

@@ -29,10 +29,14 @@ def fail_count(self) -> int:
2929
[
3030
1
3131
for test in self.test_cases
32-
if test.result.is_executed and test.result.failed
32+
if isinstance(test.result, Result) and test.result.failed
3333
]
3434
)
3535

3636
@property
3737
def skip_count(self) -> int:
38-
return len([1 for test in self.test_cases if not test.result.is_executed])
38+
return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)])
39+
40+
@property
41+
def timedout_count(self) -> int:
42+
return len([1 for test in self.test_cases if isinstance(test.result, TimedoutResult)])

‎test-runner/wasi_test_runner/test_suite_runner.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import os
44
import re
55
import shutil
6+
import subprocess
67
import time
78

89
from datetime import datetime
@@ -12,6 +13,8 @@
1213
from .runtime_adapter import RuntimeAdapter
1314
from .test_case import (
1415
Result,
16+
SkippedResult,
17+
TimedoutResult,
1518
Config,
1619
Output,
1720
TestCase,
@@ -67,7 +70,7 @@ def _skip_single_test(
6770
return TestCase(
6871
name=os.path.splitext(os.path.basename(test_path))[0],
6972
config=config,
70-
result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
73+
result=SkippedResult(),
7174
duration_s=0,
7275
)
7376

@@ -77,13 +80,17 @@ def _execute_single_test(
7780
) -> TestCase:
7881
config = _read_test_config(test_path)
7982
test_start = time.time()
80-
test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
83+
try:
84+
test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
85+
result=_validate(validators, config, test_output)
86+
except subprocess.TimeoutExpired:
87+
result=TimedoutResult()
8188
elapsed = time.time() - test_start
8289

8390
return TestCase(
8491
name=os.path.splitext(os.path.basename(test_path))[0],
8592
config=config,
86-
result=_validate(validators, config, test_output),
93+
result=result,
8794
duration_s=elapsed,
8895
)
8996

0 commit comments

Comments
 (0)