From 72f58b73137fe3076cdc91d89da39aca7880e0f5 Mon Sep 17 00:00:00 2001 From: BethanyG Date: Fri, 11 Jul 2025 17:49:34 -0700 Subject: [PATCH] Changes to tooling and tests for moving to Python 3.13 Fix requirements-generator file to fix markupsafe to 3.0.2 Upgraded version of tomli. copied content repo ci scripts over to test content Removed refs to python 3.7 Changed order of checkout action and python clone. Changed order of install Changed default spec path Relative path under github workspaces fuller path for requirements.txt Changed houskeeping job name to Python Content Validation. Removed template script from ci since it overlaps the other scripts already called. Removed docker-compose file and commented template_status on why it is not being used. Deleted run-exercise-tests-in-docker. Unneded. --- .github/workflows/main.yml | 47 +- Dockerfile | 2 +- bin/.flake8 | 13 + bin/data.py | 380 +++++++++++ bin/generate_tests.py | 458 ++++++++++++++ bin/githelp.py | 39 ++ bin/template_status.py | 147 +++++ bin/test_exercises.py | 221 +++++++ requirements-generator.txt | 12 + requirements.txt | 6 +- runner/__init__.py | 11 +- .../results.json | 400 ++++++------ .../results.json | 102 ++- test/example-empty-file/results.json | 10 +- .../example-has-stdout-and-tasks/results.json | 15 +- .../results.json | 591 +++++++++--------- test/example-has-stdout/results.json | 15 +- .../results.json | 97 ++- .../results.json | 96 +++ .../results.json | 96 +++ .../results.json | 204 +++++- test/example-success/results.json | 12 +- test/example-syntax-error/results.json | 12 +- .../results.json | 24 +- test/example-with-config/results.json | 12 +- 25 files changed, 2428 insertions(+), 594 deletions(-) create mode 100644 bin/.flake8 create mode 100644 bin/data.py create mode 100755 bin/generate_tests.py create mode 100644 bin/githelp.py create mode 100755 bin/template_status.py create mode 100755 bin/test_exercises.py create mode 100644 requirements-generator.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea5b7ce..9116f62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,10 +16,10 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 with: install: true @@ -35,3 +35,46 @@ jobs: - name: Run Tests in Docker run: bin/run-tests-in-docker.sh + + housekeeping: + name: Python Content Validation + runs-on: ubuntu-24.04 + container: + image: python:3.13.5-alpine3.22 + steps: + - name: Checkout Runner Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: Checkout Problem Specs + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + repository: exercism/problem-specifications + ref: main + path: 'problem-specifications' + + - name: Checkout Exercises + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + repository: exercism/python + ref: main + path: 'python' + + - name: Install Dependencies + run: | + pip install -r requirements-generator.txt + + - name: Generate Tests from Templates + run: | + bin/generate_tests.py --verbose -p ./problem-specifications --check + + - name: Lint Practice Exercises and Data with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./python/exercises/practice --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./python/exercises/practice --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Check Exercise Examples against Tests Using Test Runner + run: | + cd python + ../bin/test_exercises.py --runner test-runner diff --git a/Dockerfile b/Dockerfile index bc6a6f0..b386620 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.5-alpine3.18 +FROM python:3.13.5-alpine3.22 COPY requirements.txt /requirements.txt diff --git a/bin/.flake8 b/bin/.flake8 new file mode 100644 index 0000000..ad014ce --- /dev/null +++ b/bin/.flake8 @@ -0,0 +1,13 @@ +[flake8] +exclude = + .git, + __pycache__, + test, + bin, + problem-specifications, + python/bin, + python/exercises/concept, + runner, + old, + build, + dist \ No newline at end of file diff --git a/bin/data.py b/bin/data.py new file mode 100644 index 0000000..de8d279 --- /dev/null +++ b/bin/data.py @@ -0,0 +1,380 @@ +from enum import Enum +from dataclasses import dataclass, asdict, fields +import dataclasses +from itertools import chain +import json +from pathlib import Path +from typing import List, Any, Dict, Type + +# Tomli was subsumed into Python 3.11.x, but was renamed to to tomllib. +# This avoids ci failures for Python < 3.11.2. +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + + +def _custom_dataclass_init(self, *args, **kwargs): + # print(self.__class__.__name__, "__init__") + names = [field.name for field in fields(self)] + used_names = set() + + # Handle positional arguments + for value in args: + try: + name = names.pop(0) + except IndexError: + raise TypeError(f"__init__() given too many positional arguments") + # print(f'setting {k}={v}') + setattr(self, name, value) + used_names.add(name) + + # Handle keyword arguments + for name, value in kwargs.items(): + if name in names: + # print(f'setting {k}={v}') + setattr(self, name, value) + used_names.add(name) + elif name in used_names: + raise TypeError(f"__init__() got multiple values for argument '{name}'") + else: + raise TypeError( + f"Unrecognized field '{name}' for dataclass {self.__class__.__name__}." + "\nIf this field is valid, please add it to the dataclass in data.py." + "\nIf adding an object-type field, please create a new dataclass for it." + ) + + # Check for missing positional arguments + missing = [ + f"'{field.name}'" for field in fields(self) + if isinstance(field.default, dataclasses._MISSING_TYPE) and field.name not in used_names + ] + if len(missing) == 1: + raise TypeError(f"__init__() missing 1 required positional argument: {missing[0]}") + elif len(missing) == 2: + raise TypeError(f"__init__() missing 2 required positional arguments: {' and '.join(missing)}") + elif len(missing) != 0: + missing[-1] = f"and {missing[-1]}" + raise TypeError(f"__init__() missing {len(missing)} required positional arguments: {', '.join(missing)}") + + # Run post init if available + if hasattr(self, "__post_init__"): + self.__post_init__() + + +@dataclass +class TrackStatus: + __init__ = _custom_dataclass_init + + concept_exercises: bool = False + test_runner: bool = False + representer: bool = False + analyzer: bool = False + + +class IndentStyle(str, Enum): + Space = "space" + Tab = "tab" + + +@dataclass +class TestRunnerSettings: + average_run_time: float = -1 + + +@dataclass +class EditorSettings: + __init__ = _custom_dataclass_init + + indent_style: IndentStyle = IndentStyle.Space + indent_size: int = 4 + ace_editor_language: str = "python" + highlightjs_language: str = "python" + + def __post_init__(self): + if isinstance(self.indent_style, str): + self.indent_style = IndentStyle(self.indent_style) + + +class ExerciseStatus(str, Enum): + Active = "active" + WIP = "wip" + Beta = "beta" + Deprecated = "deprecated" + + +@dataclass +class ExerciseFiles: + __init__ = _custom_dataclass_init + + solution: List[str] + test: List[str] + editor: List[str] = None + exemplar: List[str] = None + + + # practice exercises are different + example: List[str] = None + + def __post_init__(self): + if self.exemplar is None: + if self.example is None: + raise ValueError( + "exercise config must have either files.exemplar or files.example" + ) + else: + self.exemplar = self.example + delattr(self, "example") + elif self.example is not None: + raise ValueError( + "exercise config must have either files.exemplar or files.example, but not both" + ) + + +@dataclass +class ExerciseConfig: + __init__ = _custom_dataclass_init + + files: ExerciseFiles + authors: List[str] = None + forked_from: str = None + contributors: List[str] = None + language_versions: List[str] = None + test_runner: bool = True + source: str = None + source_url: str = None + blurb: str = None + icon: str = None + + def __post_init__(self): + if isinstance(self.files, dict): + self.files = ExerciseFiles(**self.files) + for attr in ["authors", "contributors", "language_versions"]: + if getattr(self, attr) is None: + setattr(self, attr, []) + + @classmethod + def load(cls, config_file: Path) -> "ExerciseConfig": + with config_file.open() as f: + return cls(**json.load(f)) + + +@dataclass +class ExerciseInfo: + __init__ = _custom_dataclass_init + + path: Path + slug: str + name: str + uuid: str + prerequisites: List[str] + type: str = "practice" + status: ExerciseStatus = ExerciseStatus.Active + + # concept only + concepts: List[str] = None + + # practice only + difficulty: int = 1 + topics: List[str] = None + practices: List[str] = None + + def __post_init__(self): + if self.concepts is None: + self.concepts = [] + if self.topics is None: + self.topics = [] + if self.practices is None: + self.practices = [] + if isinstance(self.status, str): + self.status = ExerciseStatus(self.status) + + @property + def solution_stub(self): + return next( + ( + p + for p in self.path.glob("*.py") + if not p.name.endswith("_test.py") and p.name != "example.py" + ), + None, + ) + + @property + def helper_file(self): + return next(self.path.glob("*_data.py"), None) + + @property + def test_file(self): + return next(self.path.glob("*_test.py"), None) + + @property + def meta_dir(self): + return self.path / ".meta" + + @property + def exemplar_file(self): + if self.type == "concept": + return self.meta_dir / "exemplar.py" + return self.meta_dir / "example.py" + + @property + def template_path(self): + return self.meta_dir / "template.j2" + + @property + def config_file(self): + return self.meta_dir / "config.json" + + def load_config(self) -> ExerciseConfig: + return ExerciseConfig.load(self.config_file) + + +@dataclass +class Exercises: + __init__ = _custom_dataclass_init + + concept: List[ExerciseInfo] + practice: List[ExerciseInfo] + foregone: List[str] = None + + def __post_init__(self): + if self.foregone is None: + self.foregone = [] + for attr_name in ["concept", "practice"]: + base_path = Path("exercises") / attr_name + setattr( + self, + attr_name, + [ + ( + ExerciseInfo(path=(base_path / e["slug"]), type=attr_name, **e) + if isinstance(e, dict) + else e + ) + for e in getattr(self, attr_name) + ], + ) + + def all(self, status_filter={ExerciseStatus.Active, ExerciseStatus.Beta}): + return [ + e for e in chain(self.concept, self.practice) if e.status in status_filter + ] + + +@dataclass +class Concept: + __init__ = _custom_dataclass_init + + uuid: str + slug: str + name: str + + +@dataclass +class Feature: + __init__ = _custom_dataclass_init + + title: str + content: str + icon: str + + +@dataclass +class FilePatterns: + __init__ = _custom_dataclass_init + + solution: List[str] + test: List[str] + example: List[str] + exemplar: List[str] + editor: List[str] = None + + + +@dataclass +class Config: + __init__ = _custom_dataclass_init + + language: str + slug: str + active: bool + status: TrackStatus + blurb: str + version: int + online_editor: EditorSettings + exercises: Exercises + concepts: List[Concept] + key_features: List[Feature] = None + tags: List[Any] = None + test_runner: TestRunnerSettings = None + files: FilePatterns = None + + def __post_init__(self): + if isinstance(self.status, dict): + self.status = TrackStatus(**self.status) + if isinstance(self.online_editor, dict): + self.online_editor = EditorSettings(**self.online_editor) + if isinstance(self.test_runner, dict): + self.test_runner = TestRunnerSettings(**self.test_runner) + if isinstance(self.exercises, dict): + self.exercises = Exercises(**self.exercises) + if isinstance(self.files, dict): + self.files = FilePatterns(**self.files) + self.concepts = [ + (Concept(**c) if isinstance(c, dict) else c) for c in self.concepts + ] + if self.key_features is None: + self.key_features = [] + if self.tags is None: + self.tags = [] + + @classmethod + def load(cls, path="/__w/python-test-runner/python-test-runner/python/config.json"): + try: + with Path(path).open() as f: + return cls(**json.load(f)) + except IOError: + print(f"FAIL: {path} file not found") + raise SystemExit(1) + except TypeError as ex: + print(f"FAIL: {ex}") + raise SystemExit(1) + + +@dataclass +class TestCaseTOML: + __init__ = _custom_dataclass_init + + uuid: str + description: str + include: bool = True + comment: str = '' + + +@dataclass +class TestsTOML: + __init__ = _custom_dataclass_init + + cases: Dict[str, TestCaseTOML] + + @classmethod + def load(cls, toml_path: Path): + with toml_path.open("rb") as f: + data = tomllib.load(f) + return cls({uuid: TestCaseTOML(uuid, *opts) for + uuid, opts in + data.items() if + opts.get('include', None) is not False}) + + +if __name__ == "__main__": + + class CustomEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Path): + return str(obj) + return json.JSONEncoder.default(self, obj) + + config = Config.load() + print(json.dumps(asdict(config), cls=CustomEncoder, indent=2)) diff --git a/bin/generate_tests.py b/bin/generate_tests.py new file mode 100755 index 0000000..adf65b1 --- /dev/null +++ b/bin/generate_tests.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +""" +Generates exercise test suites using an exercise's canonical-data.json +(found in problem-specifications) and $exercise/.meta/template.j2. +If either does not exist, generation will not be attempted. + +Usage: + generate_tests.py Generates tests for all exercises + generate_tests.py two-fer Generates tests for two-fer exercise + generate_tests.py t* Generates tests for all exercises matching t* + + generate_tests.py --check Checks if test files are out of sync with templates + generate_tests.py --check two-fer Checks if two-fer test file is out of sync with template +""" +import sys + +from githelp import Repo + +_py = sys.version_info +if _py.major < 3 or (_py.major == 3 and _py.minor < 9): + print("Python version must be at least 3.9") + sys.exit(1) + +import argparse +from datetime import datetime +from datetime import timezone +import difflib +import filecmp +import importlib.util +import json +import logging +from pathlib import Path, PurePath, PureWindowsPath +import re +import shutil +from itertools import repeat +from string import punctuation, whitespace +from subprocess import check_call +from tempfile import NamedTemporaryFile +from textwrap import wrap +from typing import Any, Dict, List, NoReturn, Union + +# Tomli was subsumed into Python 3.11.x, but was renamed to to tomllib. +# This avoids ci failures for Python < 3.11.2. +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + +from jinja2 import Environment, FileSystemLoader, TemplateNotFound, UndefinedError +from dateutil.parser import parse + +from githelp import clone_if_missing, Repo +from data import TestsTOML + +VERSION = "0.3.0" + +TypeJSON = Dict[str, Any] + +PROBLEM_SPEC_REPO = "https://github.com/exercism/problem-specifications.git" +DEFAULT_SPEC_LOCATION = Path(".problem-specifications") +RGX_WORDS = re.compile(r"[-_\s]|(?=[A-Z])") + +logging.basicConfig() +logger = logging.getLogger("generator") +logger.setLevel(logging.WARN) + + +def replace_all(string: str, chars: Union[str, List[str]], rep: str) -> str: + """ + Replace any char in chars with rep, reduce runs and strip terminal ends. + """ + trans = str.maketrans(dict(zip(chars, repeat(rep)))) + return re.sub("{0}+".format(re.escape(rep)), rep, string.translate(trans)).strip( + rep + ) + + +def to_snake(string: str, wordchars_only: bool = False) -> str: + """ + Convert pretty much anything to to_snake. + + By default whitespace and punctuation will be converted + to underscores as well, pass wordchars_only=True to preserve these as is. + """ + clean = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", string) + clean = re.sub("([a-z0-9])([A-Z])", r"\1_\2", clean).lower() + return clean if wordchars_only else replace_all(clean, whitespace + punctuation, "_") + + +def camel_case(string: str) -> str: + """ + Convert pretty much anything to CamelCase. + """ + return "".join(w.title() for w in to_snake(string).split("_")) + + +def wrap_overlong(string: str, width: int = 70) -> List[str]: + """ + Break an overly long string literal into escaped lines. + """ + return ["{0!r} \\".format(w) for w in wrap(string, width)] + + +def parse_datetime(string: str, strip_module: bool = False) -> datetime: + """ + Parse a (hopefully ISO 8601) datestamp to a datetime object and + return its repr for use in a jinja2 template. + + If used the template will need to import the datetime module. + + import datetime + + However if strip_module is True then the template will need to + import the datetime _class_ instead. + + from datetime import datetime + """ + result = repr(parse(string)) + if strip_module: + return result.replace("datetime.", "", 1) + return result + +INVALID_ESCAPE_RE = re.compile( + r""" + \\(?! # a backslash NOT followed by + newline # the literal newline + |[ # OR precisely one of + \\ # another backslash + ' # the single quote + " # the double quote + a # the ASCII bell + b # the ASCII backspace + f # the ASCII formfeed + n # the ASCII linefeed + r # the ASCII carriage return + t # the ASCII horizontal tab + v # the ASCII vertical tab + ]| # OR + o(?:[0-8]{1,3}) # an octal value + | # OR + x(?:[0-9A-Fa-f]{2}) # a hexadecimal value + | # OR + N # a unicode char name composed of + \{ # an opening brace + [A-Z][A-Z\ \-]*[A-Z] # uppercase WORD, WORDs (or WORD-WORDs) + \} # and a closing brace + | # OR + u(?:[0-9A-Fa-f]{4}) # a 16-bit unicode char + | # OR + U(?:[0-9A-Fa-f]{8}) # a 32-bit unicode char + )""", flags=re.VERBOSE) + +def escape_invalid_escapes(string: str) -> str: + """ + Some canonical data includes invalid escape sequences, which + need to be properly escaped before template render. + """ + return INVALID_ESCAPE_RE.sub(r"\\\\", string) + +ALL_VALID = r"\newline\\\'\"\a\b\f\n\r\t\v\o123" \ + r"\xFF\N{GREATER-THAN SIGN}\u0394\U00000394" + +assert ALL_VALID == escape_invalid_escapes(ALL_VALID) + +def get_tested_properties(spec: TypeJSON) -> List[str]: + """ + Get set of tested properties from spec. Include nested cases. + """ + props = set() + for case in spec["cases"]: + if "property" in case: + props.add(case["property"]) + if "cases" in case: + props.update(get_tested_properties(case)) + return sorted(props) + + +def error_case(case: TypeJSON) -> bool: + return ( + "expected" in case + and isinstance(case["expected"], dict) + and "error" in case["expected"] + ) + + +def has_error_case(cases: List[TypeJSON]) -> bool: + cases = cases[:] + while cases: + case = cases.pop(0) + if error_case(case): + return True + cases.extend(case.get("cases", [])) + return False + + +def regex_replace(s: str, find: str, repl: str) -> str: + return re.sub(find, repl, s) + + +def regex_find(s: str, find: str) -> List[Any]: + return re.findall(find, s) + + +def regex_split(s: str, find: str) -> List[str]: + return re.split(find, s) + +def join_test_inputs(test_inputs: list) -> str: + return "\n".join(test_inputs) + +def filter_test_cases(cases: List[TypeJSON], opts: TestsTOML) -> List[TypeJSON]: + """ + Returns a filtered copy of `cases` where only cases whose UUID is marked True in + `opts` are included. + """ + filtered = [] + for case in cases: + if "uuid" in case: + uuid = case["uuid"] + case_opts = opts.cases.get(uuid, None) + if case_opts is not None and case_opts.include: + filtered.append(case) + else: + logger.debug(f"uuid {uuid} either missing or not marked for include") + elif "cases" in case: + subfiltered = filter_test_cases(case["cases"], opts) + if subfiltered: + case_copy = dict(case) + case_copy["cases"] = subfiltered + filtered.append(case_copy) + return filtered + + +def load_canonical(exercise: str, spec_path: Path, test_opts: TestsTOML) -> TypeJSON: + """ + Loads the canonical data for an exercise as a nested dictionary + """ + full_path = spec_path / "exercises" / exercise / "canonical-data.json" + with full_path.open() as f: + spec = json.load(f) + spec["cases"] = filter_test_cases(spec["cases"], test_opts) + spec["properties"] = get_tested_properties(spec) + return spec + + +def load_additional_tests(exercise: Path) -> List[TypeJSON]: + """ + Loads additional tests from .meta/additional_tests.json + """ + full_path = exercise / ".meta/additional_tests.json" + try: + with full_path.open() as f: + data = json.load(f) + return data.get("cases", []) + except FileNotFoundError: + return [] + + +def format_file(path: Path) -> NoReturn: + """ + Runs black auto-formatter on file at path + """ + check_call(["black", "-q", path]) + + +def check_template(slug: str, tests_path: Path, tmpfile: Path): + """Generate a new test file and diff against existing file. + + Note: The timestamp in each test file creates issues with + Python difflib, so it is skipped when being prepped + for diff. + + You can see this "skipping" on lines 281 & 283. + However, this rather crude method creates + an empty "false positive" diff. This empty diff is + then skipped in lines 293 & 294, so that it can be + considered a pass.. + """ + + try: + check_ok = True + if not tmpfile.is_file(): + logger.debug(f"{slug}: tmp file {tmpfile} not found") + check_ok = False + if not tests_path.is_file(): + logger.debug(f"{slug}: tests file {tests_path} not found") + check_ok = False + if check_ok and not filecmp.cmp(tmpfile, tests_path): + with tests_path.open() as f: + current_lines = f.readlines()[3:] + with tmpfile.open() as f: + rendered_lines = f.readlines()[3:] + + diff = list(difflib.unified_diff( + current_lines, + rendered_lines, + fromfile=f"[current] {tests_path.name}", + tofile=f"[generated] {tmpfile.name}", + lineterm="\n", + )) + if not diff: + check_ok = True + else: + logger.debug(f"{slug}: ##### DIFF START #####") + for line in diff: + logger.debug(line.strip()) + logger.debug(f"{slug}: ##### DIFF END #####") + check_ok = False + if not check_ok: + logger.error( + f"{slug}: check failed; tests must be regenerated with bin/generate_tests.py" + ) + return False + logger.debug(f"{slug}: check passed") + finally: + logger.debug(f"{slug}: removing tmp file {tmpfile}") + tmpfile.unlink() + return True + + +def generate_exercise(env: Environment, spec_path: Path, exercise: Path, check: bool = False): + """ + Renders test suite for exercise and if check is: + True: verifies that current tests file matches rendered + False: saves rendered to tests file + """ + slug = exercise.name + meta_dir = exercise / ".meta" + plugins_module = None + plugins_name = "plugins" + plugins_source = meta_dir / f"{plugins_name}.py" + try: + if plugins_source.is_file(): + plugins_spec = importlib.util.spec_from_file_location( + plugins_name, plugins_source + ) + plugins_module = importlib.util.module_from_spec(plugins_spec) + sys.modules[plugins_name] = plugins_module + plugins_spec.loader.exec_module(plugins_module) + try: + test_opts = TestsTOML.load(meta_dir / "tests.toml") + except FileNotFoundError: + logger.error(f"{slug}: tests.toml not found; skipping.") + return True + + spec = load_canonical(slug, spec_path, test_opts) + additional_tests = load_additional_tests(exercise) + spec["additional_cases"] = additional_tests + template_path = exercise.relative_to("exercises") / ".meta/template.j2" + + # See https://github.com/pallets/jinja/issues/767 for why this is needed on Windows systems. + if "\\" in str(template_path): + template_path = PureWindowsPath(template_path).as_posix() + + template = env.get_template(str(template_path)) + tests_path = exercise / f"{to_snake(slug)}_test.py" + spec["has_error_case"] = has_error_case(spec["cases"]) + + if plugins_module is not None: + spec[plugins_name] = plugins_module + logger.debug(f"{slug}: attempting render") + rendered = template.render(**spec) + with NamedTemporaryFile("w", delete=False) as tmp: + logger.debug(f"{slug}: writing render to tmp file {tmp.name}") + tmpfile = Path(tmp.name) + tmp.write(rendered) + try: + logger.debug(f"{slug}: formatting tmp file {tmpfile}") + format_file(tmpfile) + except FileNotFoundError as e: + logger.error(f"{slug}: the black utility must be installed") + return False + + if check: + return check_template(slug, tests_path, tmpfile) + else: + logger.debug(f"{slug}: moving tmp file {tmpfile}->{tests_path}") + shutil.move(tmpfile, tests_path) + print(f"{slug} generated at {tests_path}") + except (TypeError, UndefinedError, SyntaxError) as e: + logger.debug(str(e)) + logger.error(f"{slug}: generation failed") + return False + except TemplateNotFound as e: + logger.debug(str(e)) + logger.info(f"{slug}: no template found; skipping") + except FileNotFoundError as e: + logger.debug(str(e)) + logger.info(f"{slug}: no canonical data found; skipping") + return True + + +def generate( + exercise_glob: str, + spec_path: Path = DEFAULT_SPEC_LOCATION, + stop_on_failure: bool = False, + check: bool = False, + **_, +): + """ + Primary entry point. Generates test files for all exercises matching exercise_glob + """ + # black must be installed or all test files will error + if not shutil.which("black"): + logger.error("the black utility must be installed") + sys.exit(1) + loader = FileSystemLoader(["config", "exercises"]) + env = Environment(loader=loader, keep_trailing_newline=True) + env.filters["to_snake"] = to_snake + env.filters["camel_case"] = camel_case + env.filters["wrap_overlong"] = wrap_overlong + env.filters["regex_replace"] = regex_replace + env.filters["regex_find"] = regex_find + env.filters["regex_split"] = regex_split + env.filters["join_test_inputs"] = join_test_inputs + env.filters["zip"] = zip + env.filters["parse_datetime"] = parse_datetime + env.filters["escape_invalid_escapes"] = escape_invalid_escapes + env.globals["current_date"] = datetime.now(tz=timezone.utc).date() + env.tests["error_case"] = error_case + result = True + for exercise in sorted(Path("exercises/practice").glob(exercise_glob)): + if not generate_exercise(env, spec_path, exercise, check): + result = False + if stop_on_failure: + break + if not result: + sys.exit(1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("exercise_glob", nargs="?", default="*", metavar="EXERCISE") + parser.add_argument( + "--version", + action="version", + version="%(prog)s {} for Python {}".format(VERSION, sys.version.split("\n")[0]), + ) + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument( + "-p", + "--spec-path", + default=DEFAULT_SPEC_LOCATION, + type=Path, + help=( + "path to clone of exercism/problem-specifications " "(default: %(default)s)" + ), + ) + parser.add_argument("--stop-on-failure", action="store_true") + parser.add_argument( + "--check", + action="store_true", + help="check if tests are up-to-date, but do not modify test files", + ) + opts = parser.parse_args() + if opts.verbose: + logger.setLevel(logging.DEBUG) + with clone_if_missing(repo=Repo.ProblemSpecifications, directory=opts.spec_path): + generate(**opts.__dict__) diff --git a/bin/githelp.py b/bin/githelp.py new file mode 100644 index 0000000..3d3be06 --- /dev/null +++ b/bin/githelp.py @@ -0,0 +1,39 @@ +from contextlib import contextmanager +from enum import Enum +from pathlib import Path +import shutil +import subprocess +from typing import Iterator, Union + + +GITHUB_EXERCISM = f"https://github.com/exercism" + + +class Repo(Enum): + ProblemSpecifications = f"{GITHUB_EXERCISM}/problem-specifications.git" + + + +def clone(repo: Union[str, Repo], directory: Union[str, Path, None] = None) -> bool: + if isinstance(repo, Repo): + repo = repo.value + if directory is None: + directory = repo.split("/")[-1].split(".")[0] + directory = Path(directory) + if not directory.is_dir(): + try: + subprocess.run(["git", "clone", repo, str(directory)], check=True) + return True + except subprocess.CalledProcessError: + pass + return False + + +@contextmanager +def clone_if_missing(repo: Union[str, Repo], directory: Union[str, Path, None] = None) -> Iterator[None]: + temp_clone = clone(repo, directory) + try: + yield directory + finally: + if temp_clone: + shutil.rmtree(directory) diff --git a/bin/template_status.py b/bin/template_status.py new file mode 100755 index 0000000..3620f12 --- /dev/null +++ b/bin/template_status.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# This script is currently unused for this ropo. +# It's content essentially doubles that of the other content test scripts, +# but it only tests practice exercises and fails out if any of them +# are missing templates. It needs to be modified to gracefully ignore +# (or otherwise test) those exercises that do not have canonical data. +# We're leaving it here as a potential starting place for +# further checks or testing. + + +import argparse +from argparse import Namespace +from enum import IntEnum, auto +from fnmatch import fnmatch +import logging +from pathlib import Path +import shlex +from subprocess import check_call, DEVNULL, CalledProcessError +import sys +from typing import List, Iterator + +from data import Config, ExerciseInfo, ExerciseStatus +from generate_tests import clone_if_missing +from githelp import Repo +from test_exercises import check_assignment + +PROBLEM_SPEC_REPO = "https://github.com/exercism/problem-specifications.git" +DEFAULT_SPEC_LOCATION = Path(".problem-specifications") + +logging.basicConfig(format="%(levelname)s:%(message)s") +logger = logging.getLogger("generator") +logger.setLevel(logging.WARN) + + +class TemplateStatus(IntEnum): + OK = auto() + MISSING = auto() + INVALID = auto() + TEST_FAILURE = auto() + + +def exec_cmd(cmd: str) -> bool: + try: + args = shlex.split(cmd) + if logger.isEnabledFor(logging.DEBUG): + check_call(args) + else: + check_call(args, stderr=DEVNULL, stdout=DEVNULL) + return True + except CalledProcessError as e: + logger.debug(str(e)) + return False + + +def generate_template(exercise: ExerciseInfo, spec_path: Path) -> bool: + script = Path('bin/generate_tests.py') + return exec_cmd(f'{script} --verbose --spec-path "{spec_path}" {exercise.slug}') + + +def run_tests(exercise: ExerciseInfo) -> bool: + return check_assignment(exercise, quiet=True) == 0 + + +def get_status(exercise: ExerciseInfo, spec_path: Path) -> TemplateStatus: + if exercise.template_path.is_file(): + if generate_template(exercise, spec_path): + if run_tests(exercise): + logging.info(f"{exercise.slug}: OK") + return TemplateStatus.OK + else: + return TemplateStatus.TEST_FAILURE + else: + return TemplateStatus.INVALID + else: + return TemplateStatus.MISSING + + +def set_loglevel(opts: Namespace): + if opts.quiet: + logger.setLevel(logging.FATAL) + elif opts.verbose >= 2: + logger.setLevel(logging.DEBUG) + elif opts.verbose >= 1: + logger.setLevel(logging.INFO) + + +def filter_exercises(exercises: List[ExerciseInfo], pattern: str) -> Iterator[ExerciseInfo]: + for exercise in exercises: + if exercise.status != ExerciseStatus.Deprecated: + if exercise.type == 'concept': + # Concept exercises are not generated + continue + if fnmatch(exercise.slug, pattern): + yield exercise + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("exercise_pattern", nargs="?", default="*", metavar="EXERCISE") + parser.add_argument("-v", "--verbose", action="count", default=0) + parser.add_argument("-q", "--quiet", action="store_true") + parser.add_argument("--stop-on-failure", action="store_true") + parser.add_argument( + "-p", + "--spec-path", + default=DEFAULT_SPEC_LOCATION, + type=Path, + help=( + "path to clone of exercism/problem-specifications " "(default: %(default)s)" + ), + ) + opts = parser.parse_args() + set_loglevel(opts) + + if not opts.spec_path.is_dir(): + logger.error(f"{opts.spec_path} is not a directory") + sys.exit(1) + with clone_if_missing(repo=Repo.ProblemSpecifications, directory=opts.spec_path): + + result = True + buckets = { + TemplateStatus.MISSING: [], + TemplateStatus.INVALID: [], + TemplateStatus.TEST_FAILURE: [], + } + config = Config.load() + for exercise in filter_exercises(config.exercises.all(), "*"): + status = get_status(exercise, opts.spec_path) + if status == TemplateStatus.OK: + logger.info(f"{exercise.slug}: {status.name}") + else: + buckets[status].append(exercise.slug) + result = False + if opts.stop_on_failure: + logger.error(f"{exercise.slug}: {status.name}") + break + + if not opts.quiet and not opts.stop_on_failure: + for status, bucket in sorted(buckets.items()): + if bucket: + print(f"The following exercises have status '{status.name}'") + for exercise in sorted(bucket): + print(f' {exercise}') + + if not result: + sys.exit(1) diff --git a/bin/test_exercises.py b/bin/test_exercises.py new file mode 100755 index 0000000..d66d4cd --- /dev/null +++ b/bin/test_exercises.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Meant to be run from inside python-test-runner container, +where this track repo is mounted at /python +""" +import argparse +from functools import wraps +from itertools import zip_longest +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from typing import List +from data import Config, ExerciseConfig, ExerciseInfo, ExerciseStatus + +# Allow high-performance tests to be skipped +ALLOW_SKIP = ['alphametics', 'largest-series-product'] + +TEST_RUNNER_DIR = Path('/__w/python-test-runner/python-test-runner/') + +RUNNERS = {} + + +def runner(name): + def _decorator(runner_func): + RUNNERS[name] = runner_func + @wraps(runner_func) + def _wrapper(exercise: ExerciseInfo, workdir: Path, quiet: bool = False): + return runner_func(exercise, workdir, quiet=quiet) + return _wrapper + return _decorator + + +def copy_file(src: Path, dst: Path, strip_skips=False): + if strip_skips: + with src.open('r') as src_file: + lines = [line for line in src_file.readlines() + if not line.strip().startswith('@unittest.skip')] + with dst.open('w') as dst_file: + dst_file.writelines(lines) + else: + shutil.copy2(src, dst) + +def copy_solution_files(exercise: ExerciseInfo, workdir: Path, exercise_config: ExerciseConfig = None): + if exercise_config is not None: + solution_files = exercise_config.files.solution + exemplar_files = exercise_config.files.exemplar + helper_files = exercise_config.files.editor + else: + solution_files = [] + exemplar_files = [] + helper_files = [] + + if helper_files: + helper_files = [exercise.path / h for h in helper_files] + for helper_file in helper_files: + dst = workdir / helper_file.relative_to(exercise.path) + copy_file(helper_file, dst) + + if not solution_files: + solution_files.append(exercise.solution_stub.name) + solution_files = [exercise.path / s for s in solution_files] + if not exemplar_files: + exemplar_files.append(exercise.exemplar_file.relative_to(exercise.path)) + exemplar_files = [exercise.path / e for e in exemplar_files] + + for solution_file, exemplar_file in zip_longest(solution_files, exemplar_files): + if solution_file is None: + copy_file(exemplar_file, workdir / exemplar_file.name) + elif exemplar_file is None: + copy_file(solution_file, workdir / solution_file.name) + else: + dst = workdir / solution_file.relative_to(exercise.path) + copy_file(exemplar_file, dst) + + +def copy_test_files(exercise: ExerciseInfo, workdir: Path, exercise_config = None): + if exercise_config is not None: + test_files = exercise_config.files.test + helper_files = exercise_config.files.editor + else: + test_files = [] + helper_files = [] + + if helper_files: + for helper_file_name in helper_files: + helper_file = exercise.path / helper_file_name + helper_file_out = workdir / helper_file_name + copy_file(helper_file, helper_file_out, strip_skips=(exercise.slug not in ALLOW_SKIP)) + + if not test_files: + test_files.append(exercise.test_file.name) + + for test_file_name in test_files: + test_file = exercise.path / test_file_name + test_file_out = workdir / test_file_name + copy_file(test_file, test_file_out, strip_skips=(exercise.slug not in ALLOW_SKIP)) + + +def copy_exercise_files(exercise: ExerciseInfo, workdir: Path): + exercise_config = None + if exercise.config_file.is_file(): + workdir_meta = workdir / '.meta' + workdir_meta.mkdir(exist_ok=True) + copy_file(exercise.config_file, workdir_meta / exercise.config_file.name) + exercise_config = exercise.load_config() + copy_solution_files(exercise, workdir, exercise_config) + copy_test_files(exercise, workdir, exercise_config) + + +@runner('pytest') +def run_with_pytest(_exercise, workdir, quiet: bool = False) -> int: + kwargs = {'cwd': str(workdir)} + if quiet: + kwargs['stdout'] = subprocess.DEVNULL + kwargs['stderr'] = subprocess.DEVNULL + return subprocess.run([sys.executable, '-m', 'pytest'], **kwargs).returncode + + +@runner('test-runner') +def run_with_test_runner(exercise, workdir, quiet: bool = False) -> int: + kwargs = {} + if quiet: + kwargs['stdout'] = subprocess.DEVNULL + kwargs['stderr'] = subprocess.DEVNULL + if TEST_RUNNER_DIR.is_dir(): + kwargs['cwd'] = str(TEST_RUNNER_DIR) + args = ['./bin/run.sh', exercise.slug, workdir, workdir] + else: + args = [ + 'docker compose', + 'run', + '-w', str(TEST_RUNNER_DIR), + '--entrypoint', './bin/run.sh', + '-v', f'{workdir}:/{exercise.slug}', + 'test-runner', + exercise.slug, + f'/{exercise.slug}', + f'/{exercise.slug}', + ] + subprocess.run(args, **kwargs) + results_file = workdir / 'results.json' + if results_file.is_file(): + with results_file.open() as f: + results = json.load(f) + if results['status'] == 'pass': + return 0 + return 1 + + +def check_assignment(exercise: ExerciseInfo, runner: str = 'pytest', quiet: bool = False) -> int: + ret = 1 + with tempfile.TemporaryDirectory(exercise.slug) as workdir: + workdir = Path(workdir) + copy_exercise_files(exercise, workdir) + ret = RUNNERS[runner](exercise, workdir, quiet=quiet) + return ret + + +def get_cli() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + runners = list(RUNNERS.keys()) + if not runners: + print('No runners registered!') + raise SystemExit(1) + parser.add_argument('-q', '--quiet', action='store_true') + parser.add_argument('--deprecated', action='store_true', help='include deprecated exercises', dest='include_deprecated') + parser.add_argument('--wip', action='store_true', help='include WIP exercises', dest='include_wip') + parser.add_argument('-r', '--runner', choices=runners, default=runners[0]) + parser.add_argument('exercises', nargs='*') + return parser + + +def main(): + opts = get_cli().parse_args() + config = Config.load() + status_filter = {ExerciseStatus.Active, ExerciseStatus.Beta} + if opts.include_deprecated: + status_filter.add(ExerciseStatus.Deprecated) + if opts.include_wip: + status_filter.add(ExerciseStatus.WIP) + exercises = config.exercises.all(status_filter) + if opts.exercises: + # test specific exercises + exercises = [ + e for e in exercises if e.slug in opts.exercises + ] + not_found = [ + slug for slug in opts.exercises + if not any(e.slug == slug for e in exercises) + ] + if not_found: + for slug in not_found: + if slug not in exercises: + print(f"unknown or disabled exercise '{slug}'") + raise SystemExit(1) + + print(f'TestEnvironment: {sys.executable.capitalize()}') + print(f'Runner: {opts.runner}\n\n') + + failures = [] + for exercise in exercises: + print('# ', exercise.slug) + if not exercise.test_file: + print('FAIL: File with test cases not found') + failures.append('{} (FileNotFound)'.format(exercise.slug)) + else: + if check_assignment(exercise, runner=opts.runner, quiet=opts.quiet): + failures.append('{} (TestFailed)'.format(exercise.slug)) + print('') + + if failures: + print('FAILURES: ', ', '.join(failures)) + raise SystemExit(1) + else: + print('SUCCESS!') + + +if __name__ == "__main__": + main() diff --git a/requirements-generator.txt b/requirements-generator.txt new file mode 100644 index 0000000..db7cfa3 --- /dev/null +++ b/requirements-generator.txt @@ -0,0 +1,12 @@ +Jinja2~=3.1.6 +black<=25.1.0 +flake8~=7.3.0 +markupsafe==3.0.2 +pytest-subtests~=0.14.2 +pytest~=8.4.0 +python-dateutil==2.8.1 +requests~=2.32.4 +tomli>=2.2.1; python_full_version < '3.11.2' + + + diff --git a/requirements.txt b/requirements.txt index e748954..6711558 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -black<=22.3.0 -pytest~=7.2.2 -pytest-subtests~=0.11.0 +black<=25.1.0 +pytest~=8.4.0 +pytest-subtests~=0.14.2 tomli>=1.1.0; python_full_version < '3.11.2' diff --git a/runner/__init__.py b/runner/__init__.py index 80c07b6..fc43073 100644 --- a/runner/__init__.py +++ b/runner/__init__.py @@ -101,9 +101,7 @@ def pytest_runtest_logreport(self, report): state.output = report.capstdout return - else: - if report.capstdout: - state.output = report.capstdout + # Handle details of test failure if report.failed: @@ -134,7 +132,12 @@ def pytest_runtest_logreport(self, report): # Changes status of parent to fail if any of the subtests fail. - if state.fail: + if state.is_passing(): + self.tests[parent_test_name].status = state.status + + self.tests[parent_test_name].test_code = state.test_code + + else: self.tests[parent_test_name].fail( message="One or more variations of this test failed. Details can be found under each [variant#]." ) diff --git a/test/example-all-fail-tasks-and-subtests/results.json b/test/example-all-fail-tasks-and-subtests/results.json index ea67d2c..9864d9f 100644 --- a/test/example-all-fail-tasks-and-subtests/results.json +++ b/test/example-all-fail-tasks-and-subtests/results.json @@ -1,202 +1,202 @@ { - "version": 3, - "status": "fail", - "tests": [ - { - "name": "ExampleAllFail > abc [variation #1]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #2]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #3]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #4]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #5]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #6]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #7]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc [variation #8]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > abc~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #1]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #2]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #3]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #4]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFailOther > dummy [variation #1]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #2]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #3]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #4]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #5]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #6]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #7]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy [variation #8]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > dummy~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > hello [variation #1]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > hello [variation #2]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > hello [variation #3]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > hello [variation #4]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - }, - { - "name": "ExampleAllFailOther > hello~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 - } - ] + "version":3, + "status":"fail", + "tests":[ + { + "name":"ExampleAllFail > abc [variation #1]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #2]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #3]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #4]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #5]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #6]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #7]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc [variation #8]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > abc~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > hello [variation #1]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", + "test_code":"input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > hello [variation #2]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", + "test_code":"input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > hello [variation #3]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", + "test_code":"input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > hello [variation #4]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", + "test_code":"input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFail > hello~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":1 + }, + { + "name":"ExampleAllFailOther > dummy [variation #1]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #2]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #3]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #4]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #5]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #6]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #7]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy [variation #8]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > dummy~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > hello [variation #1]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > hello [variation #2]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > hello [variation #3]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > hello [variation #4]", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + }, + { + "name":"ExampleAllFailOther > hello~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id":2 + } + ] } \ No newline at end of file diff --git a/test/example-config-multiple-files-subtests-and-tasks/results.json b/test/example-config-multiple-files-subtests-and-tasks/results.json index 77a5c2d..1aa2025 100644 --- a/test/example-config-multiple-files-subtests-and-tasks/results.json +++ b/test/example-config-multiple-files-subtests-and-tasks/results.json @@ -6,57 +6,127 @@ "name": "ExampleFirst > abc~", "status": "pass", "test_code": "self.assertEqual(hello(13), (\"Hello, World!\", 13))", - "task_id": 1, - "output": "User output is captured! 13" + "task_id": 1 }, { "name": "ExampleFirst > hello~", "status": "pass", "test_code": "self.assertEqual(hello('Hi'), (\"Hello, World!\", 'Hi'))", - "task_id": 1, - "output": "User output is captured! Hi" + "task_id": 1 }, { "name": "ExampleFirstOther > dummy~", "status": "pass", "test_code": "self.assertEqual(hello('Banana'), (\"Hello, World!\", \"Banana\"))", - "task_id": 2, - "output": "User output is captured! Banana" + "task_id": 2 }, { "name": "ExampleFirstOther > hello~", "status": "pass", "test_code": "self.assertEqual(hello(42), (\"Hello, World!\", 42))", - "task_id": 2, - "output": "User output is captured! 42" + "task_id": 2 + }, + { + "name": "ExampleSecond > abc [variation #1]", + "status": "pass", + "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 + }, + { + "name": "ExampleSecond > abc [variation #2]", + "status": "pass", + "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 + }, + { + "name": "ExampleSecond > abc [variation #3]", + "status": "pass", + "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 + }, + { + "name": "ExampleSecond > abc [variation #4]", + "status": "pass", + "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 }, { "name": "ExampleSecond > abc~", "status": "pass", "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 3, - "output": "User output is captured! carrot\nUser output is captured! cucumber\nUser output is captured! grass\nUser output is captured! tree" + "task_id": 3 + }, + { + "name": "ExampleSecond > hello [variation #1]", + "status": "pass", + "test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 + }, + { + "name": "ExampleSecond > hello [variation #2]", + "status": "pass", + "test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 + }, + { + "name": "ExampleSecond > hello [variation #3]", + "status": "pass", + "test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 3 }, { "name": "ExampleSecond > hello~", "status": "pass", "test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 3, - "output": "User output is captured! 1\nUser output is captured! 2\nUser output is captured! 5" + "task_id": 3 + }, + { + "name": "ExampleSecondOther > dummy [variation #1]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 + }, + { + "name": "ExampleSecondOther > dummy [variation #2]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 + }, + { + "name": "ExampleSecondOther > dummy [variation #3]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 }, { "name": "ExampleSecondOther > dummy~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "User output is captured! frog\nUser output is captured! fish\nUser output is captured! coconut" + "task_id": 4 + }, + { + "name": "ExampleSecondOther > hello [variation #1]", + "status": "pass", + "test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 + }, + { + "name": "ExampleSecondOther > hello [variation #2]", + "status": "pass", + "test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 + }, + { + "name": "ExampleSecondOther > hello [variation #3]", + "status": "pass", + "test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 }, { "name": "ExampleSecondOther > hello~", "status": "pass", "test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "User output is captured! 23\nUser output is captured! 33\nUser output is captured! 39" + "task_id": 4 } ] } \ No newline at end of file diff --git a/test/example-empty-file/results.json b/test/example-empty-file/results.json index 413ca2c..5832abb 100644 --- a/test/example-empty-file/results.json +++ b/test/example-empty-file/results.json @@ -1,6 +1,8 @@ { - "version": 3, - "status": "error", - "message": " ImportError while importing test module './test/example-empty-file/example_empty_file_test.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/local/lib/python3.11/importlib/__init__.py:126: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\ntest/example-empty-file/example_empty_file_test.py:4: in \n from example_empty_file import hello\nE ImportError: cannot import name 'hello' from 'example_empty_file' (./test/example-empty-file/example_empty_file.py)", - "tests": [] + "version":3, + "status":"error", + "message":" ImportError while importing test module './test/example-empty-file/example_empty_file_test.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/usr/local/lib/python3.13/importlib/__init__.py:88: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ntest/example-empty-file/example_empty_file_test.py:4: in \n from example_empty_file import hello\nE ImportError: cannot import name 'hello' from 'example_empty_file' (./test/example-empty-file/example_empty_file.py)", + "tests":[ + + ] } \ No newline at end of file diff --git a/test/example-has-stdout-and-tasks/results.json b/test/example-has-stdout-and-tasks/results.json index 7d77261..e8ba113 100644 --- a/test/example-has-stdout-and-tasks/results.json +++ b/test/example-has-stdout-and-tasks/results.json @@ -7,40 +7,35 @@ "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 1, - "output": "Hello, World!" + "task_id": 1 }, { "name": "ExampleHasStdoutAndTasks > abc~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 2, - "output": "Hello, World!" + "task_id": 2 }, { "name": "ExampleHasStdoutAndTasks > truncation~", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")", - "task_id": 3, - "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" + "task_id": 3 }, { "name": "ExampleHasStdoutAndTasksOther > dummy~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 4, - "output": "Hello, World!" + "task_id": 4 }, { "name": "ExampleHasStdoutAndTasksOther > hello~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 5, - "output": "Hello, World!" + "task_id": 5 } ] } \ No newline at end of file diff --git a/test/example-has-stdout-tasks-and-subtests/results.json b/test/example-has-stdout-tasks-and-subtests/results.json index 245b0d9..4e90ad8 100644 --- a/test/example-has-stdout-tasks-and-subtests/results.json +++ b/test/example-has-stdout-tasks-and-subtests/results.json @@ -1,298 +1,297 @@ { - "version": 3, - "status": "fail", - "tests": [ - { - "name": "ExampleHasStdout > hello [variation #1]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 1" - }, - { - "name": "ExampleHasStdout > hello [variation #2]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 2" - }, - { - "name": "ExampleHasStdout > hello [variation #3]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 5" - }, - { - "name": "ExampleHasStdout > hello [variation #4]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 10" - }, - { - "name": "ExampleHasStdout > hello [variation #5]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 15" - }, - { - "name": "ExampleHasStdout > hello [variation #6]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 23" - }, - { - "name": "ExampleHasStdout > hello [variation #7]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 33" - }, - { - "name": "ExampleHasStdout > hello [variation #8]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "Hello, World! 39" - }, - { - "name": "ExampleHasStdout > hello~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1 - }, - { - "name": "ExampleHasStdout > abc [variation #1]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! frog" - }, - { - "name": "ExampleHasStdout > abc [variation #2]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! fish" - }, - { - "name": "ExampleHasStdout > abc [variation #3]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! coconut" - }, - { - "name": "ExampleHasStdout > abc [variation #4]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! pineapple" - }, - { - "name": "ExampleHasStdout > abc [variation #5]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! carrot" - }, - { - "name": "ExampleHasStdout > abc [variation #6]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! cucumber" - }, - { - "name": "ExampleHasStdout > abc [variation #7]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! grass" - }, - { - "name": "ExampleHasStdout > abc [variation #8]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "Hello, World! tree" - }, - { - "name": "ExampleHasStdout > abc~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2 - }, - { - "name": "ExampleHasStdout > truncation~", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", - "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")", - "task_id": 3, - "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #1]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! frog" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #2]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! fish" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #3]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! coconut" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #4]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! pineapple" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #5]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! carrot" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #6]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! cucumber" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #7]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! grass" - }, - { - "name": "ExampleHasStdoutOther > dummy [variation #8]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4, - "output": "Hello, World! tree" - }, - { - "name": "ExampleHasStdoutOther > dummy~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4 - }, - { - "name": "ExampleHasStdoutOther > hello [variation #1]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 1" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #2]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 2" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #3]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 5" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #4]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 10" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #5]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 15" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #6]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 23" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #7]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 33" - }, - { - "name": "ExampleHasStdoutOther > hello [variation #8]", - "status": "fail", - "message": "AssertionError: None != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5, - "output": "Hello, World! 39" - }, - { - "name": "ExampleHasStdoutOther > hello~", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5 - } - ] + "version":3, + "status":"fail", + "tests":[ + { + "name":"ExampleHasStdout > hello [variation #1]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 1" + }, + { + "name":"ExampleHasStdout > hello [variation #2]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 2" + }, + { + "name":"ExampleHasStdout > hello [variation #3]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 5" + }, + { + "name":"ExampleHasStdout > hello [variation #4]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 10" + }, + { + "name":"ExampleHasStdout > hello [variation #5]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 15" + }, + { + "name":"ExampleHasStdout > hello [variation #6]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 23" + }, + { + "name":"ExampleHasStdout > hello [variation #7]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 33" + }, + { + "name":"ExampleHasStdout > hello [variation #8]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1, + "output":"Hello, World! 39" + }, + { + "name":"ExampleHasStdout > hello~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":1 + }, + { + "name":"ExampleHasStdout > abc [variation #1]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! frog" + }, + { + "name":"ExampleHasStdout > abc [variation #2]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! fish" + }, + { + "name":"ExampleHasStdout > abc [variation #3]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! coconut" + }, + { + "name":"ExampleHasStdout > abc [variation #4]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! pineapple" + }, + { + "name":"ExampleHasStdout > abc [variation #5]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! carrot" + }, + { + "name":"ExampleHasStdout > abc [variation #6]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! cucumber" + }, + { + "name":"ExampleHasStdout > abc [variation #7]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! grass" + }, + { + "name":"ExampleHasStdout > abc [variation #8]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2, + "output":"Hello, World! tree" + }, + { + "name":"ExampleHasStdout > abc~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":2 + }, + { + "name":"ExampleHasStdout > truncation~", + "status":"fail", + "message":"AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", + "test_code":"self.assertEqual(must_truncate(), \"Hello, World!\")", + "task_id":3 + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #1]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'frog') : Expected: ('Hello, World!', 'frog') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! frog" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #2]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'fish') : Expected: ('Hello, World!', 'fish') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! fish" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #3]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'coconut') : Expected: ('Hello, World!', 'coconut') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! coconut" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #4]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'pineapple') : Expected: ('Hello, World!', 'pineapple') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! pineapple" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #5]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'carrot') : Expected: ('Hello, World!', 'carrot') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! carrot" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #6]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'cucumber') : Expected: ('Hello, World!', 'cucumber') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! cucumber" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #7]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'grass') : Expected: ('Hello, World!', 'grass') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! grass" + }, + { + "name":"ExampleHasStdoutOther > dummy [variation #8]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 'tree') : Expected: ('Hello, World!', 'tree') but got something else instead.", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4, + "output":"Hello, World! tree" + }, + { + "name":"ExampleHasStdoutOther > dummy~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":4 + }, + { + "name":"ExampleHasStdoutOther > hello [variation #1]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 1" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #2]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 2" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #3]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 5" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #4]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 10" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #5]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 15" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #6]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 23" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #7]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 33" + }, + { + "name":"ExampleHasStdoutOther > hello [variation #8]", + "status":"fail", + "message":"AssertionError: None != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5, + "output":"Hello, World! 39" + }, + { + "name":"ExampleHasStdoutOther > hello~", + "status":"fail", + "message":"One or more variations of this test failed. Details can be found under each [variant#].", + "test_code":"input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id":5 + } + ] } \ No newline at end of file diff --git a/test/example-has-stdout/results.json b/test/example-has-stdout/results.json index 9a22c7b..020aeed 100644 --- a/test/example-has-stdout/results.json +++ b/test/example-has-stdout/results.json @@ -7,40 +7,35 @@ "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "Hello, World!" + "task_id": 0 }, { "name": "ExampleHasStdout > abc", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "Hello, World!" + "task_id": 0 }, { "name": "ExampleHasStdout > truncation", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")", - "task_id": 0, - "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" + "task_id": 0 }, { "name": "ExampleHasStdoutOther > dummy", "status": "fail", "message": "AssertionError: 'hello, world.' != 'Hello, World!'\n- hello, world.\n? ^ ^ ^\n+ Hello, World!\n? ^ ^ ^", "test_code": "self.assertEqual(other(), \"Hello, World!\")", - "task_id": 0, - "output": "Hello, World!\n5 6 7" + "task_id": 0 }, { "name": "ExampleHasStdoutOther > hello", "status": "fail", "message": "AssertionError: 'hello, world.' != 'Hello, World!'\n- hello, world.\n? ^ ^ ^\n+ Hello, World!\n? ^ ^ ^", "test_code": "self.assertEqual(other(), \"Hello, World!\")", - "task_id": 0, - "output": "Hello, World!\n5 6 7" + "task_id": 0 } ] } \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests-and-stdout-2/results.json b/test/example-partial-failure-with-subtests-and-stdout-2/results.json index 1754274..af70243 100644 --- a/test/example-partial-failure-with-subtests-and-stdout-2/results.json +++ b/test/example-partial-failure-with-subtests-and-stdout-2/results.json @@ -2,36 +2,120 @@ "version": 3, "status": "fail", "tests": [ + { + "name": "CurrencyExchange > exchange money [variation #1]", + "status": "pass", + "test_code": "test_data = [(100000, 0.8), (700000, 10.0)]\nresult_data = [125000, 70000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n expected=expected):\n\n actual_result = exchange_money(*params)\n error_message = (f'Called exchange_money{budget, exchange_rate}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} when exchanging'\n f' {budget} at a rate of {exchange_rate}.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 1 + }, + { + "name": "CurrencyExchange > exchange money [variation #2]", + "status": "pass", + "test_code": "test_data = [(100000, 0.8), (700000, 10.0)]\nresult_data = [125000, 70000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n expected=expected):\n\n actual_result = exchange_money(*params)\n error_message = (f'Called exchange_money{budget, exchange_rate}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} when exchanging'\n f' {budget} at a rate of {exchange_rate}.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 1 + }, { "name": "CurrencyExchange > exchange money~", "status": "pass", "test_code": "test_data = [(100000, 0.8), (700000, 10.0)]\nresult_data = [125000, 70000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n expected=expected):\n\n actual_result = exchange_money(*params)\n error_message = (f'Called exchange_money{budget, exchange_rate}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} when exchanging'\n f' {budget} at a rate of {exchange_rate}.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", "task_id": 1 }, + { + "name": "CurrencyExchange > get change [variation #1]", + "status": "pass", + "test_code": "test_data = [(463000, 5000), (1250, 120), (15000, 1380)]\nresult_data = [458000, 1130, 13620]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchanging_value = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchanging_value=exchanging_value,\n expected=expected):\n\n actual_result = get_change(*params)\n error_message = (f'Called get_change{budget, exchanging_value}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} left in your budget.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 2 + }, + { + "name": "CurrencyExchange > get change [variation #2]", + "status": "pass", + "test_code": "test_data = [(463000, 5000), (1250, 120), (15000, 1380)]\nresult_data = [458000, 1130, 13620]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchanging_value = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchanging_value=exchanging_value,\n expected=expected):\n\n actual_result = get_change(*params)\n error_message = (f'Called get_change{budget, exchanging_value}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} left in your budget.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 2 + }, + { + "name": "CurrencyExchange > get change [variation #3]", + "status": "pass", + "test_code": "test_data = [(463000, 5000), (1250, 120), (15000, 1380)]\nresult_data = [458000, 1130, 13620]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchanging_value = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchanging_value=exchanging_value,\n expected=expected):\n\n actual_result = get_change(*params)\n error_message = (f'Called get_change{budget, exchanging_value}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} left in your budget.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 2 + }, { "name": "CurrencyExchange > get change~", "status": "pass", "test_code": "test_data = [(463000, 5000), (1250, 120), (15000, 1380)]\nresult_data = [458000, 1130, 13620]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchanging_value = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchanging_value=exchanging_value,\n expected=expected):\n\n actual_result = get_change(*params)\n error_message = (f'Called get_change{budget, exchanging_value}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} left in your budget.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", "task_id": 2 }, + { + "name": "CurrencyExchange > get value of bills [variation #1]", + "status": "pass", + "test_code": "test_data = [(10000, 128), (50, 360), (200, 200)]\nresult_data = [1280000, 18000, 40000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n denomination, number_of_bills = params\n\n with self.subTest(f\"variation #{variant}\",\n denomination=denomination,\n number_of_bills=number_of_bills,\n expected=expected):\n\n actual_result = get_value_of_bills(*params)\n error_message = (f'Called get_value_of_bills{denomination, number_of_bills}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} for the bills value.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 3 + }, + { + "name": "CurrencyExchange > get value of bills [variation #2]", + "status": "pass", + "test_code": "test_data = [(10000, 128), (50, 360), (200, 200)]\nresult_data = [1280000, 18000, 40000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n denomination, number_of_bills = params\n\n with self.subTest(f\"variation #{variant}\",\n denomination=denomination,\n number_of_bills=number_of_bills,\n expected=expected):\n\n actual_result = get_value_of_bills(*params)\n error_message = (f'Called get_value_of_bills{denomination, number_of_bills}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} for the bills value.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 3 + }, + { + "name": "CurrencyExchange > get value of bills [variation #3]", + "status": "pass", + "test_code": "test_data = [(10000, 128), (50, 360), (200, 200)]\nresult_data = [1280000, 18000, 40000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n denomination, number_of_bills = params\n\n with self.subTest(f\"variation #{variant}\",\n denomination=denomination,\n number_of_bills=number_of_bills,\n expected=expected):\n\n actual_result = get_value_of_bills(*params)\n error_message = (f'Called get_value_of_bills{denomination, number_of_bills}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} for the bills value.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 3 + }, { "name": "CurrencyExchange > get value of bills~", "status": "pass", "test_code": "test_data = [(10000, 128), (50, 360), (200, 200)]\nresult_data = [1280000, 18000, 40000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n denomination, number_of_bills = params\n\n with self.subTest(f\"variation #{variant}\",\n denomination=denomination,\n number_of_bills=number_of_bills,\n expected=expected):\n\n actual_result = get_value_of_bills(*params)\n error_message = (f'Called get_value_of_bills{denomination, number_of_bills}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} for the bills value.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", "task_id": 3 }, + { + "name": "CurrencyExchange > get number of bills [variation #1]", + "status": "pass", + "test_code": "test_data = [(163270, 50000), (54361, 1000)]\nresult_data = [3, 54]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_number_of_bills(amount, denomination)\n error_message = (f'Called get_number_of_bills{amount, denomination}. '\n f'The function returned {actual_result} bills, but '\n f'The tests expected {expected} bills.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 4 + }, + { + "name": "CurrencyExchange > get number of bills [variation #2]", + "status": "pass", + "test_code": "test_data = [(163270, 50000), (54361, 1000)]\nresult_data = [3, 54]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_number_of_bills(amount, denomination)\n error_message = (f'Called get_number_of_bills{amount, denomination}. '\n f'The function returned {actual_result} bills, but '\n f'The tests expected {expected} bills.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 4 + }, { "name": "CurrencyExchange > get number of bills~", "status": "pass", "test_code": "test_data = [(163270, 50000), (54361, 1000)]\nresult_data = [3, 54]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_number_of_bills(amount, denomination)\n error_message = (f'Called get_number_of_bills{amount, denomination}. '\n f'The function returned {actual_result} bills, but '\n f'The tests expected {expected} bills.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", "task_id": 4 }, + { + "name": "CurrencyExchange > get leftover of bills [variation #1]", + "status": "pass", + "test_code": "test_data = [(10.1, 10), (654321.0, 5), (3.14, 2)]\nresult_data = [0.1, 1.0, 1.14]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_leftover_of_bills(*params)\n error_message = (f'Called get_leftover_of_bills{amount, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the leftover amount.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 5 + }, + { + "name": "CurrencyExchange > get leftover of bills [variation #2]", + "status": "pass", + "test_code": "test_data = [(10.1, 10), (654321.0, 5), (3.14, 2)]\nresult_data = [0.1, 1.0, 1.14]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_leftover_of_bills(*params)\n error_message = (f'Called get_leftover_of_bills{amount, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the leftover amount.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 5 + }, + { + "name": "CurrencyExchange > get leftover of bills [variation #3]", + "status": "pass", + "test_code": "test_data = [(10.1, 10), (654321.0, 5), (3.14, 2)]\nresult_data = [0.1, 1.0, 1.14]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_leftover_of_bills(*params)\n error_message = (f'Called get_leftover_of_bills{amount, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the leftover amount.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 5 + }, { "name": "CurrencyExchange > get leftover of bills~", "status": "pass", "test_code": "test_data = [(10.1, 10), (654321.0, 5), (3.14, 2)]\nresult_data = [0.1, 1.0, 1.14]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_leftover_of_bills(*params)\n error_message = (f'Called get_leftover_of_bills{amount, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the leftover amount.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", - "task_id": 5, - "output": "denomination is:: 10\nnumber of bills: 1\ndenomination is:: 5\nnumber of bills: 130864\ndenomination is:: 2\nnumber of bills: 1" + "task_id": 5 + }, + { + "name": "CurrencyExchange > exchangeable value [variation #1]", + "status": "pass", + "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 6, + "output": "params :: budget=100000, exchange_rate=10.61, spread=10, denomination=1\nfee is: 1.061\nrate is 11.671\namount is:: 8568.246080027418\n,params :: budget=1500, exchange_rate=0.84, spread=25, denomination=40\nfee is: 0.21\nrate is 1.05\namount is:: 1428.5714285714284" }, { "name": "CurrencyExchange > exchangeable value [variation #2]", @@ -39,7 +123,14 @@ "message": "AssertionError: 35 != 1400 : Called exchangeable_value(1500, 0.84, 25, 40). The function returned 35, but The tests expected 1400 as the maximum value of the new currency .", "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", "task_id": 6, - "output": "params :: budget=470000, exchange_rate=1050, spread=30, denomination=10000000000\nfee is: 315.0\nrate is 1365.0\namount is:: 344.32234432234435\nparams :: budget=470000, exchange_rate=9e-08, spread=30, denomination=700\nfee is: 2.7e-08\nrate is 1.17e-07\namount is:: 4017094017094.017" + "output": "params :: budget=470000, exchange_rate=1050, spread=30, denomination=10000000000\nfee is: 315.0\nrate is 1365.0\namount is:: 344.32234432234435\n,params :: budget=470000, exchange_rate=9e-08, spread=30, denomination=700\nfee is: 2.7e-08\nrate is 1.17e-07\namount is:: 4017094017094.017" + }, + { + "name": "CurrencyExchange > exchangeable value [variation #3]", + "status": "pass", + "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 6, + "output": "params :: budget=425.33, exchange_rate=0.0009, spread=30, denomination=700\nfee is: 0.00027\nrate is 0.00117\namount is:: 363529.9145299145" }, { "name": "CurrencyExchange > exchangeable value [variation #4]", diff --git a/test/example-partial-failure-with-subtests-and-stdout/results.json b/test/example-partial-failure-with-subtests-and-stdout/results.json index 3d59ed4..7631747 100644 --- a/test/example-partial-failure-with-subtests-and-stdout/results.json +++ b/test/example-partial-failure-with-subtests-and-stdout/results.json @@ -2,6 +2,102 @@ "version": 3, "status": "fail", "tests": [ + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #10]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #11]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #12]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #13]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #14]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #15]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #16]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #1]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #2]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #3]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #4]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #5]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #6]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #7]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #8]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes [variation #9]", + "status": "pass", + "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 1 + }, { "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes~", "status": "pass", diff --git a/test/example-partial-failure-with-subtests/results.json b/test/example-partial-failure-with-subtests/results.json index 84ab759..958a097 100644 --- a/test/example-partial-failure-with-subtests/results.json +++ b/test/example-partial-failure-with-subtests/results.json @@ -2,6 +2,54 @@ "version": 3, "status": "fail", "tests": [ + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #1]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #2]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #3]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #4]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #5]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #6]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #7]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtests > abc [variation #8]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, { "name": "ExamplePartialFailureWithSubtests > abc~", "status": "pass", @@ -43,6 +91,54 @@ "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 1 }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #1]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #2]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #3]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #4]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #5]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #6]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #7]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy [variation #8]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, { "name": "ExamplePartialFailureWithSubtestsOther > dummy~", "status": "pass", diff --git a/test/example-success-with-subtests/results.json b/test/example-success-with-subtests/results.json index d4571fd..74e971c 100644 --- a/test/example-success-with-subtests/results.json +++ b/test/example-success-with-subtests/results.json @@ -2,33 +2,221 @@ "version": 3, "status": "pass", "tests": [ + { + "name": "ExampleSuccessWithSubtests > abc [variation #1]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #2]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #3]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #4]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #5]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #6]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #7]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > abc [variation #8]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, { "name": "ExampleSuccessWithSubtests > abc~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #1]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #2]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #3]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #4]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #5]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #6]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #7]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtests > hello [variation #8]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 }, { "name": "ExampleSuccessWithSubtests > hello~", "status": "pass", "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1, - "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" + "task_id": 1 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #1]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #2]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #3]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #4]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #5]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #6]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #7]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > dummy [variation #8]", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 }, { "name": "ExampleSuccessWithSubtestsOther > dummy~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #1]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #2]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #3]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #4]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #5]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #6]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #7]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleSuccessWithSubtestsOther > hello [variation #8]", + "status": "pass", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 }, { "name": "ExampleSuccessWithSubtestsOther > hello~", "status": "pass", "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2, - "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" + "task_id": 2 } ] } \ No newline at end of file diff --git a/test/example-success/results.json b/test/example-success/results.json index b936e1e..2645959 100644 --- a/test/example-success/results.json +++ b/test/example-success/results.json @@ -6,29 +6,25 @@ "name": "ExampleSuccess > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSuccess > abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSuccessOther > dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSuccessOther > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 } ] } \ No newline at end of file diff --git a/test/example-syntax-error/results.json b/test/example-syntax-error/results.json index 8c8c0fc..7974e51 100644 --- a/test/example-syntax-error/results.json +++ b/test/example-syntax-error/results.json @@ -1,6 +1,8 @@ { - "version": 3, - "status": "error", - "message": " /usr/local/lib/python3.11/site-packages/_pytest/python.py:618: in _importtestmodule\n mod = import_path(self.path, mode=importmode, root=self.config.rootpath)\n/usr/local/lib/python3.11/site-packages/_pytest/pathlib.py:533: in import_path\n importlib.import_module(module_name)\n/usr/local/lib/python3.11/importlib/__init__.py:126: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n:1204: in _gcd_import\n ???\n:1176: in _find_and_load\n ???\n:1147: in _find_and_load_unlocked\n ???\n:690: in _load_unlocked\n ???\n/usr/local/lib/python3.11/site-packages/_pytest/assertion/rewrite.py:168: in exec_module\n exec(co, module.__dict__)\ntest/example-syntax-error/example_syntax_error_test.py:4: in \n from example_syntax_error import hello\nE File \"./test/example-syntax-error/example_syntax_error.py\", line 3\nE def hello();\nE ^\nE SyntaxError: expected ':'", - "tests": [] -} + "version":3, + "status":"error", + "message":" /usr/local/lib/python3.13/site-packages/_pytest/python.py:498: in importtestmodule\n mod = import_path(\n/usr/local/lib/python3.13/site-packages/_pytest/pathlib.py:587: in import_path\n importlib.import_module(module_name)\n/usr/local/lib/python3.13/importlib/__init__.py:88: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/usr/local/lib/python3.13/site-packages/_pytest/assertion/rewrite.py:186: in exec_module\n exec(co, module.__dict__)\ntest/example-syntax-error/example_syntax_error_test.py:4: in \n from example_syntax_error import hello\nE File \"./test/example-syntax-error/example_syntax_error.py\", line 3\nE def hello();\nE ^\nE SyntaxError: expected ':'", + "tests":[ + + ] +} \ No newline at end of file diff --git a/test/example-with-config-multiple-files/results.json b/test/example-with-config-multiple-files/results.json index b3311a1..2206efd 100644 --- a/test/example-with-config-multiple-files/results.json +++ b/test/example-with-config-multiple-files/results.json @@ -6,57 +6,49 @@ "name": "ExampleFirst > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSecond > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleFirst > abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSecond > abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleFirstOther > dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSecondOther > dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleFirstOther > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleSecondOther > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 } ] } \ No newline at end of file diff --git a/test/example-with-config/results.json b/test/example-with-config/results.json index 222bdc9..409f476 100644 --- a/test/example-with-config/results.json +++ b/test/example-with-config/results.json @@ -6,29 +6,25 @@ "name": "ExampleWithConfig > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleWithConfig > abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleWithConfigOther > dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 }, { "name": "ExampleWithConfigOther > hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", - "task_id": 0, - "output": "User output is captured!" + "task_id": 0 } ] } \ No newline at end of file