diff --git a/tests/__init__.py b/tests/__init__.py index fb83adb..f50e652 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,11 @@ +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import timedelta import os import sys import subprocess +from time import perf_counter from typing import List # , Set, Dict, Tuple, Optional from tests.util import * @@ -10,7 +14,7 @@ class Test(object): - STAGES: dict = { + STAGES: dict[str, list[str]] = { "autogen": ["autogen.sh"], "configure": ["configure.sh"], "make": ["make.sh", "cmake.sh"], @@ -38,12 +42,14 @@ def print_log_tail_on_fail(script_path): if os.path.isfile(logfile): grep_cmd = ['grep', '-i', '-A', '20', '-E', 'panicked|error', logfile] grep = subprocess.Popen(grep_cmd, stdout=subprocess.PIPE) + assert grep.stdout is not None for line in grep.stdout: print(line.decode().rstrip()) # fall back to tail if grep didn't find anything if grep.returncode != 0: tail = subprocess.Popen(['tail', '-n', '20', logfile], stdout=subprocess.PIPE) + assert tail.stdout is not None for line in tail.stdout: print(line.decode().rstrip()) else: @@ -53,7 +59,6 @@ def print_log_tail_on_fail(script_path): nocolor=Colors.NO_COLOR) ) - prev_dir = os.getcwd() script_path = os.path.join(self.dir, script) if not os.path.isfile(script_path): @@ -73,14 +78,15 @@ def print_log_tail_on_fail(script_path): return False if not verbose: - relpath = os.path.relpath(script_path, prev_dir) + relpath = os.path.relpath(script_path, os.getcwd()) line = "{blue}{name}{nc}: {stage}({script})".format( blue=Colors.OKBLUE, name=self.name, nc=Colors.NO_COLOR, stage=stage, script=relpath) - print(line, end="", flush=True) + else: + line = "" # if we already have `compile_commands.json`, skip the build stages if stage in ["autogen", "configure", "make"]: @@ -93,7 +99,7 @@ def print_log_tail_on_fail(script_path): fill = (75 - len(line)) * "." color = Colors.OKBLUE msg = "OK_CACHED" - print(f"{fill} {color}{msg}{Colors.NO_COLOR}") + print(f"{line}{fill} {color}{msg}{Colors.NO_COLOR}") return True elif emsg: if verbose: @@ -103,45 +109,35 @@ def print_log_tail_on_fail(script_path): except OSError: print(f"could not remove {compile_commands}") - - success = False - # noinspection PyBroadException try: - os.chdir(self.dir) if verbose: - subprocess.check_call(args=[script_path]) + subprocess.check_call(cwd=self.dir, args=[script_path]) else: subprocess.check_call( + cwd=self.dir, args=[script_path], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) + stderr=subprocess.DEVNULL, + ) fill = (75 - len(line)) * "." color = Colors.WARNING if xfail else Colors.OKGREEN msg = "OK_XFAIL" if xfail else "OK" - print(f"{fill} {color}{msg}{Colors.NO_COLOR}") - success = True + print(f"{line}{fill} {color}{msg}{Colors.NO_COLOR}") + return True except KeyboardInterrupt: if not verbose: - print(": {color}INTERRUPT{nocolor}".format( - color=Colors.WARNING, - nocolor=Colors.NO_COLOR) - ) + print(f"{line}: {Colors.WARNING}INTERRUPT{Colors.NO_COLOR}") exit(1) except Exception: # noqa if not verbose: outcome = "XFAIL" if xfail else "FAIL" - print("{fill} {color}{outcome}{nocolor}".format( - fill=(75 - len(line)) * ".", - color=Colors.OKBLUE if xfail else Colors.FAIL, - outcome=outcome, - nocolor=Colors.NO_COLOR) - ) + fill = (75 - len(line)) * "." + color = Colors.OKBLUE if xfail else Colors.FAIL + print(f"{line}{fill} {color}{outcome}{Colors.NO_COLOR}") print_log_tail_on_fail(script_path) - finally: - os.chdir(prev_dir) - return success + return False def ensure_submodule_checkout(self): # make sure the `repo` directory exists and is not empty @@ -175,7 +171,7 @@ def has_xfail_file() -> bool: die(f"expected boolean xfail value; found {xfail}") return xfail - def __call__(self, conf: Config): + def run(self, conf: Config) -> bool: """Returns true if test was successful or expected to fail, false on unexpected failure """ @@ -183,7 +179,7 @@ def __call__(self, conf: Config): self.ensure_submodule_checkout() stages = Test.STAGES.keys() - if conf.stages: + if conf.stages is not None: # Check that all stages are valid for stage in conf.stages: if stage not in Test.STAGES: @@ -201,20 +197,37 @@ def __call__(self, conf: Config): xfail = self.is_stage_xfail(stage, script, conf) cont = self.run_script(stage, script, conf.verbose, xfail) if not cont: + print(f"{self.name} failed on stage {stage}") return xfail break # found script for stage; skip alternatives return True -def run_tests(conf): +@dataclass +class TestResult: + test: Test + passed: bool + time: timedelta + + +def run_tests(conf: Config): if not conf.ignore_requirements: check(conf) tests = [Test(td) for td in conf.project_dirs] - failure = False - for tt in tests: - failure |= not tt(conf) + def run(test: Test) -> TestResult: + start = perf_counter() + passed = test.run(conf) + end = perf_counter() + time = timedelta(seconds=end - start) + return TestResult(test=test, passed=passed, time=time) + + with ThreadPoolExecutor() as executor: + results = executor.map(run, tests) - if failure: + for result in results: + print(f"{result.test.name} took {result.time}") + if not all(result.passed for result in results): + print(f"projects failed: {" ".join(result.test.name for result in results)}") exit(1) diff --git a/tests/util.py b/tests/util.py index 9223b8b..3f84f14 100644 --- a/tests/util.py +++ b/tests/util.py @@ -4,12 +4,14 @@ import json import errno -from typing import List, Iterable +from typing import Any, List, Iterable, Never, Sequence CONF_YML: str = "conf.yml" class Config(object): + stages: list[str] | None + def __init__(self, args): self.verbose = args.verbose self.projects = args.projects # projects filter @@ -18,8 +20,8 @@ def __init__(self, args): self.project_dirs = find_project_dirs(self) self.project_conf = {cf: get_yaml(cf) for cf in get_conf_files(self)} - def try_get_conf_for(self, conf_file, *keys: List[str]): - def lookup(yaml, keys: List[str]): + def try_get_conf_for(self, conf_file, *keys: str): + def lookup(yaml, keys: Sequence[str]): if not keys: return None head, *tail = keys @@ -44,7 +46,7 @@ class Colors(object): NO_COLOR = '\033[0m' -def die(emsg: str, status: int=errno.EINVAL): +def die(emsg: str, status: int=errno.EINVAL) -> Never: (red, nc) = (Colors.FAIL, Colors.NO_COLOR) print(f"{red}error:{nc} {emsg}", file=sys.stderr) exit(status) @@ -97,7 +99,7 @@ def find_project_dirs(conf: Config) -> List[str]: return [os.path.join(script_dir, s) for s in subdirs] -def get_yaml(file: str) -> dict: +def get_yaml(file: str) -> dict[str, Any]: with open(file, 'r') as stream: try: return yaml.safe_load(stream) @@ -105,7 +107,7 @@ def get_yaml(file: str) -> dict: die(str(exc)) -def check_compile_commands(compile_commands_path: str) -> (bool, str): +def check_compile_commands(compile_commands_path: str) -> tuple[bool, str]: """ Return True iff compile_commands_path points to a valid compile_commands.json and all referenced source files exist.