diff --git a/tested/configs.py b/tested/configs.py index 0cb0cc03a..4d03da713 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -67,7 +67,7 @@ class Options: """ -@fallback_field(get_converter(), {"testplan": "test_suite", "plan_name": "test_suite"}) +@fallback_field({"testplan": "test_suite", "plan_name": "test_suite"}) @define class DodonaConfig: resources: Path diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index d9e3f4074..2ac49472b 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -276,11 +276,18 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" + "oneOf" : [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "$ref": "#/definitions/fileData" + } ] }, "arguments" : { @@ -890,6 +897,35 @@ "$ref" : "#/definitions/fileConfigurationOptions" } } + }, + "fileData": { + "type": "object", + "additionalProperties" : false, + "anyOf" : [ + { + "required" : [ + "content" + ] + }, + { + "required" : [ + "path" + ] + } + ], + "properties": { + "content": { + "type": [ + "string", + "path" + ], + "description" : "Content of the file, which will be provided inline or written to disk in the workdir. If a !path, the file contents will be read from the provided path." + }, + "path": { + "type": "string", + "description" : "Path to the file, relative to the workdir. Used to display in the output." + } + } } } } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f8d066bca..665403a25 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -276,11 +276,18 @@ }, "stdin" : { "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" + "oneOf" : [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "$ref": "#/definitions/fileData" + } ] }, "arguments" : { @@ -884,6 +891,32 @@ "$ref" : "#/definitions/fileConfigurationOptions" } } + }, + "fileData": { + "type": "object", + "additionalProperties" : false, + "anyOf" : [ + { + "required" : [ + "content" + ] + }, + { + "required" : [ + "path" + ] + } + ], + "properties": { + "content": { + "type": "string", + "description" : "Content of the file, which will be provided inline or written to disk in the workdir. If a !path, the file contents will be read from the provided path." + }, + "path": { + "type": "string", + "description" : "Path to the file, relative to the workdir. Used to display in the output." + } + } } } } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 245b0cf7d..846d566b7 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -45,6 +45,7 @@ Value, ) from tested.testsuite import ( + ContentPath, Context, CustomCheckOracle, EmptyChannel, @@ -88,9 +89,22 @@ class ReturnOracle(dict): pass +class ContentPathString(str): + pass + + OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | ContentPathString ) @@ -138,6 +152,12 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: return ReturnOracle(result) +def _return_path(loader: yaml.Loader, node: yaml.Node) -> ContentPathString: + result = _parse_yaml_value(loader, node) + assert isinstance(result, str), f"A path must be a string, got {result}" + return ContentPathString(result) + + def _parse_yaml(yaml_stream: str) -> YamlObject: """ Parse a string or stream to YAML. @@ -148,6 +168,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor("!path", _return_path, loader) try: return yaml.load(yaml_stream, loader) @@ -187,6 +208,10 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_path(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, ContentPathString) + + def load_schema_validator( dsl_object: YamlObject = None, file: str = "schema-strict.json" ) -> Validator: @@ -215,9 +240,13 @@ def validate_tested_dsl_expression(value: object) -> bool: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression) + type_checker = original_validator.TYPE_CHECKER.redefine_many( + { + "oracle": is_oracle, + "expression": is_expression, + "path": is_path, + } + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)( validate_tested_dsl_expression @@ -469,16 +498,16 @@ def _convert_text_output_channel( data = raw_data if isinstance(stream, str): - return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) + return TextOutputChannel(content=data, oracle=GenericTextOracle(options=config)) else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": return TextOutputChannel( - data=data, oracle=GenericTextOracle(options=config) + content=data, oracle=GenericTextOracle(options=config) ) elif stream["oracle"] == "custom_check": return TextOutputChannel( - data=data, oracle=_convert_custom_check_oracle(stream) + content=data, oracle=_convert_custom_check_oracle(stream) ) raise TypeError(f"Unknown text oracle type: {stream['oracle']}") @@ -599,8 +628,33 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - assert isinstance(testcase["stdin"], str) - stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) + if isinstance(testcase["stdin"], dict): + stdin_object = testcase["stdin"] + raw_content = stdin_object.get("content") + raw_path = stdin_object.get("path") + + if isinstance(raw_content, ContentPathString): + file_content = ContentPath(path=raw_content) + elif isinstance(raw_content, str): + file_content = _ensure_trailing_newline(raw_content) + elif isinstance(raw_path, str): + file_content = ContentPath(path=raw_path) + else: + assert ( + False + ), f"Invalid stdin content is required but got {type(raw_content)}" + + assert raw_path is None or isinstance( + raw_path, str + ), "Path must be a string if given." + + file_path = raw_path + else: + assert isinstance(testcase["stdin"], str) + file_content = _ensure_trailing_newline(testcase["stdin"]) + file_path = None + + stdin = TextData(content=file_content, path=file_path) else: stdin = EmptyChannel.NONE arguments = testcase.get("arguments", []) diff --git a/tested/judge/execution.py b/tested/judge/execution.py index c5ee6f42c..96c81ae72 100644 --- a/tested/judge/execution.py +++ b/tested/judge/execution.py @@ -1,5 +1,6 @@ import itertools import logging +import shutil from pathlib import Path from attrs import define @@ -16,6 +17,7 @@ ) from tested.languages.conventionalize import selector_name from tested.languages.preparation import exception_file, value_file +from tested.testsuite import ContentPath from tested.utils import safe_del _logger = logging.getLogger(__name__) @@ -180,6 +182,26 @@ def set_up_unit( # See https://github.com/dodona-edu/universal-judge/issues/57 destination.hardlink_to(origin) + # Create dynamically generated files if necessary. + dynamically_generated_file = unit.get_dynamically_generated_files() + if dynamically_generated_file is not None: + destination = execution_dir / dynamically_generated_file.path + + if isinstance(dynamically_generated_file.content, ContentPath): + _logger.debug( + f"Copying input file %s to %s", + dynamically_generated_file.content.path, + destination, + ) + source_file = ( + bundle.config.resources / dynamically_generated_file.content.path + ) + shutil.copy2(source_file, destination) + else: + _logger.debug(f"Creating dynamically generated file %s", destination) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(dynamically_generated_file.content) + return execution_dir, dependencies diff --git a/tested/judge/planning.py b/tested/judge/planning.py index 827d53857..1b50bfa06 100644 --- a/tested/judge/planning.py +++ b/tested/judge/planning.py @@ -13,7 +13,7 @@ from tested.dodona import AnnotateCode, Message, Status from tested.languages.conventionalize import execution_name from tested.languages.language import FileFilter -from tested.testsuite import Context, EmptyChannel, MainInput +from tested.testsuite import ContentPath, Context, EmptyChannel, MainInput @define @@ -33,6 +33,12 @@ class PlannedContext: context_index: int +@define(frozen=True) +class DynamicallyGeneratedFile: + path: str + content: ContentPath | str + + @define class PlannedExecutionUnit: """ @@ -56,6 +62,27 @@ def has_main_testcase(self) -> bool: def has_exit_testcase(self) -> bool: return self.contexts[-1].context.has_exit_testcase() + def get_dynamically_generated_files(self) -> DynamicallyGeneratedFile | None: + if not self.has_main_testcase(): + return None + + stdin = cast(MainInput, self.contexts[0].context.testcases[0].input).stdin + + if stdin == EmptyChannel.NONE: + return None + + if not stdin.is_dynamically_generated(): + return None + + # For type checking, in the future should be removable. + if stdin.path is None: + return None + + return DynamicallyGeneratedFile( + path=stdin.path, + content=stdin.content, + ) + @define class ExecutionPlan: diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 7c7d883b4..045b82f81 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -128,12 +128,16 @@ def get_readable_input( args = f"$ {command}" # Determine the stdin if isinstance(case.input.stdin, TextData): - stdin = case.input.stdin.get_data_as_string(bundle.config.resources) + if case.input.stdin.path is not None: + stdin = Path(case.input.stdin.path) + else: + stdin = case.input.stdin.get_data_as_string(bundle.config.resources) else: stdin = "" - # If we have both stdin and arguments, we use a here-document. - if case.input.arguments and stdin: + if stdin and isinstance(stdin, Path): + text = f"${args} < {stdin}" + elif case.input.arguments and stdin: assert stdin[-1] == "\n", "stdin must end with a newline" delimiter = _get_heredoc_token(stdin) text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" diff --git a/tested/languages/preparation.py b/tested/languages/preparation.py index ed496d80c..f7bfb4ecb 100644 --- a/tested/languages/preparation.py +++ b/tested/languages/preparation.py @@ -487,10 +487,10 @@ def prepare_context( """ language = bundle.config.programming_language resources = bundle.config.resources - before_code = context.before.get(language, TextData(data="")).get_data_as_string( + before_code = context.before.get(language, TextData(content="")).get_data_as_string( resources ) - after_code = context.after.get(language, TextData(data="")).get_data_as_string( + after_code = context.after.get(language, TextData(content="")).get_data_as_string( resources ) testcases, evaluator_names = prepare_testcases(bundle, context) diff --git a/tested/oracles/common.py b/tested/oracles/common.py index c8f67afda..3bc2dfa44 100644 --- a/tested/oracles/common.py +++ b/tested/oracles/common.py @@ -74,7 +74,6 @@ class OracleResult: @fallback_field( - get_converter(), { "readableExpected": "readable_expected", "readableActual": "readable_actual", diff --git a/tested/parsing.py b/tested/parsing.py index 3b7f3d64f..ebe2360c1 100644 --- a/tested/parsing.py +++ b/tested/parsing.py @@ -6,11 +6,9 @@ """ import logging -from collections.abc import Callable from decimal import Decimal -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast -from cattrs import Converter from cattrs.gen import make_dict_structure_fn from cattrs.preconf.json import JsonConverter, make_converter from typing_inspect import is_union_type @@ -26,6 +24,14 @@ _suite_converter = make_converter(forbid_extra_keys=True, omit_if_default=True) initialized = False +T = TypeVar("T") + +# Saves the registry of structure hooks. +# Note that this assumes we only have a single converter. +_TESTED_STRUCTURE_HOOK_REGISTRIES: dict[type, Callable[[dict[str, Any], type], Any]] = ( + dict() +) + def structure_decimal(obj: Any, _) -> Decimal: return Decimal(str(obj)) @@ -35,8 +41,9 @@ def unstructure_decimal(obj: Decimal) -> str: return str(obj) -def structure_every_union(to_convert: Any, the_type: type) -> Any: +def structure_every_union(to_convert: Any, the_type: Any) -> Any: from tested.serialisation import Identifier + from tested.testsuite import ContentPath _logger.debug(f"=== Finding type for {to_convert}, from {the_type}...") if to_convert is None and type(None) in get_args(the_type): @@ -45,6 +52,9 @@ def structure_every_union(to_convert: Any, the_type: type) -> Any: if isinstance(to_convert, bool) and bool in get_args(the_type): _logger.debug(f"Yes: found boolean: {to_convert}.") return to_convert + if isinstance(to_convert, ContentPath) and ContentPath in get_args(the_type): + _logger.debug(f"Yes: found content path: {to_convert}.") + return to_convert for possible_class in get_args(the_type): debug_message = f"{possible_class} -> " @@ -111,68 +121,96 @@ def suite_to_json(suite: "Suite") -> str: return _suite_converter.dumps(suite) -def fallback_field(converter_arg: Converter, old_to_new_field: dict[str, str]): - def decorator(cls): - struct = make_dict_structure_fn( - cls, converter_arg, _cattrs_forbid_extra_keys=False - ) - - def structure(d, cl): - for k, v in old_to_new_field.items(): - if k in d: - d[v] = d[k] +def _chain_structure_hook( + cls: type[T], + wrapper: Callable[ + [dict[str, Any], type[T], Callable[[dict[str, Any], type[T]], Any]], Any + ], +): + """ + Compose structure hooks for a class. - return struct(d, cl) + cattrs only keeps one structure hook per class; decorator stacking would otherwise + overwrite earlier hooks. We keep our own per-converter registry and wrap the + previously registered hook. - converter_arg.register_structure_hook(cls, structure) + Since the wrapper is executed first and then the previous one is called, the final + order in which the transformations are applied is top to bottom. - return cls + For example: + @fallback_field(..., 1) # top + @fallback_field(..., 2) + @ignore_field(..., "show_expected") + @ignore_field(..., "type") # bottom + class TextOutputChannel(...): - return decorator + When structuring a dictionary, the transformations will be applied in the same order: + 1. `@fallback_field(..., 1)` + 2. `@fallback_field(..., 2)` + 3. `@ignore_field(..., "show_expected")` + 4. `@ignore_field(..., "type")` + """ + converter = get_converter() -def custom_fallback_field( - converter_arg: Converter, - old_to_new_field: dict[str, tuple[str, Callable[[Any], Any]]], -): - def decorator(cls): - struct = make_dict_structure_fn( - cls, converter_arg, _cattrs_forbid_extra_keys=False + previous = _TESTED_STRUCTURE_HOOK_REGISTRIES.get(cls) + if previous is None: + previous = make_dict_structure_fn( + cls, converter, _cattrs_forbid_extra_keys=False ) - def structure(d, cl): - for k, (new_name, mapper) in old_to_new_field.items(): - if k in d: - if new_name in d: - raise ValueError( - f"You cannot use {new_name} and {k} simultaneously. Migrate to {new_name}." - ) - d[new_name] = mapper(d[k]) - - return struct(d, cl) + def composed(d: dict[str, Any], cl: type[T]): + if not isinstance(d, dict): + return previous(d, cl) - converter_arg.register_structure_hook(cls, structure) + # Work on a shallow copy so we don't mutate dicts. + d2 = dict(d) + return wrapper(d2, cl, previous) - return cls + # Python's types cannot encode dict[T, Callable[..., T]]. + # However, we know that the key is a type that the value expects, so cast it. + _TESTED_STRUCTURE_HOOK_REGISTRIES[cls] = cast( + Callable[[dict[str, Any], type], Any], composed + ) - return decorator + converter.register_structure_hook(cls, composed) -def ignore_field(converter_arg: Converter, *fields: str): +def ignore_field(*fields: str): def decorator(cls): - struct = make_dict_structure_fn( - cls, converter_arg, _cattrs_forbid_extra_keys=False - ) - - def structure(d, cl): + def _wrapper(d: dict[str, Any], cl: type, next_hook): for to_ignore in fields: - if to_ignore in d: - del d[to_ignore] + d.pop(to_ignore, None) + return next_hook(d, cl) + + _chain_structure_hook(cls, _wrapper) + return cls - return struct(d, cl) + return decorator - converter_arg.register_structure_hook(cls, structure) +def fallback_field( + old_to_new_field: dict[str, str | tuple[str, Callable[[Any, dict[str, Any]], Any]]], +): + def decorator(cls): + def _wrapper(d: dict[str, Any], cl: type, next_hook): + for old_name, mapping in old_to_new_field.items(): + if old_name in d: + if isinstance(mapping, tuple): + new_name, mapper = mapping + if new_name in d: + raise ValueError( + f"You cannot use {new_name} and {old_name} simultaneously. " + f"Migrate to {new_name}." + ) + d[new_name] = mapper(d[old_name], d) + else: + new_name = mapping + if new_name not in d: + d[new_name] = d[old_name] + return next_hook(d, cl) + + _chain_structure_hook(cls, _wrapper) return cls return decorator diff --git a/tested/testsuite.py b/tested/testsuite.py index 4ef5f0aac..9ac5ac5af 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -24,12 +24,7 @@ WithFeatures, combine_features, ) -from tested.parsing import ( - custom_fallback_field, - fallback_field, - get_converter, - ignore_field, -) +from tested.parsing import fallback_field, ignore_field from tested.serialisation import ( Expression, FunctionCall, @@ -181,7 +176,7 @@ def validate_languages(self, _, value): raise ValueError("At least one language is required.") -@fallback_field(get_converter(), {"evaluators": "functions"}) +@fallback_field({"evaluators": "functions"}) @define class LanguageSpecificOracle: """ @@ -226,12 +221,6 @@ def validate_evaluator(self, _, value): raise ValueError("At least one language-specific oracle is required.") -@unique -class TextChannelType(StrEnum): - TEXT = "text" # Literal values - FILE = "file" # Path to a file - - def _resolve_path(working_directory, file_path): """ Resolve a path to an absolute path. Relative paths will be resolved against @@ -243,30 +232,58 @@ def _resolve_path(working_directory, file_path): return path.abspath(path.join(working_directory, file_path)) +@define(frozen=True) +class ContentPath: + path: str + + +def _data_to_content_converter(data: str | None, full: Any) -> str | ContentPath | None: + if isinstance(data, str): + if full.get("type") == "file": + return ContentPath(path=data) + return data + + return data + + +@fallback_field({"data": ("content", _data_to_content_converter)}) +@ignore_field("type") @define class TextData(WithFeatures): """Describes textual data: either directly or in a file.""" - data: str - type: TextChannelType = TextChannelType.TEXT + content: ContentPath | str + + # For input files: The filename shown in the description. + # For output files: The filename the student is expected to generate. + # For stdin: The filename shown in the description (e.g. "< hello.txt"). + # For stdout/stderr: None + path: str | None = None def get_data_as_string(self, working_directory: Path) -> str: - """Get the data as a string, reading the file if necessary.""" - if self.type == TextChannelType.TEXT: - return self.data - elif self.type == TextChannelType.FILE: - file_path = _resolve_path(working_directory, self.data) + if isinstance(self.content, ContentPath): + file_path = _resolve_path(working_directory, self.content.path) with open(file_path, "r") as file: return file.read() else: - raise AssertionError(f"Unknown enum type {self.type}") + return self.content + + def is_dynamically_generated(self) -> bool: + return self.path is not None and not ( + isinstance(self.content, ContentPath) and self.content.path == self.path + ) def get_used_features(self) -> FeatureSet: return NOTHING -@fallback_field(get_converter(), {"evaluator": "oracle"}) -@ignore_field(get_converter(), "show_expected") +@fallback_field( + { + "data": ("content", _data_to_content_converter), + "evaluator": "oracle", + } +) +@ignore_field("show_expected", "type") @define class TextOutputChannel(TextData): """Describes the output for textual channels.""" @@ -274,8 +291,8 @@ class TextOutputChannel(TextData): oracle: GenericTextOracle | CustomCheckOracle = field(factory=GenericTextOracle) -@fallback_field(get_converter(), {"evaluator": "oracle"}) -@ignore_field(get_converter(), "show_expected") +@fallback_field({"evaluator": "oracle"}) +@ignore_field("show_expected") @define class FileOutputChannel(WithFeatures): """Describes the output for files.""" @@ -295,8 +312,8 @@ def get_data_as_string(self, resources: Path) -> str: return file.read() -@fallback_field(get_converter(), {"evaluator": "oracle"}) -@ignore_field(get_converter(), "show_expected") +@fallback_field({"evaluator": "oracle"}) +@ignore_field("show_expected") @define class ValueOutputChannel(WithFeatures): """Handles return values of function calls.""" @@ -357,8 +374,8 @@ def readable(self, language: SupportedLanguage) -> str: return type_ -@fallback_field(get_converter(), {"evaluator": "oracle"}) -@ignore_field(get_converter(), "show_expected") +@fallback_field({"evaluator": "oracle"}) +@ignore_field("show_expected") @define class ExceptionOutputChannel(WithFeatures): """Handles exceptions caused by the submission.""" @@ -378,7 +395,7 @@ def __attrs_post_init__(self): raise ValueError("The generic oracle needs a channel exception.") -@ignore_field(get_converter(), "show_expected") +@ignore_field("show_expected") @define class ExitCodeOutputChannel(WithFeatures): """Handles exit codes.""" @@ -533,7 +550,7 @@ class FileUrl: name: str -@ignore_field(get_converter(), "essential") +@ignore_field("essential") @define class Testcase(WithFeatures, WithFunctions): """ @@ -618,7 +635,7 @@ def get_specific_languages(self) -> set[SupportedLanguage] | None: Code = dict[str, TextData] -@ignore_field(get_converter(), "link_files") +@ignore_field("link_files") @define class Context(WithFeatures, WithFunctions): """ @@ -672,7 +689,7 @@ def get_files(self) -> set[FileUrl]: return all_files -def _runs_to_tab_converter(runs: list | None): +def _runs_to_tab_converter(runs: list | None, _): assert isinstance(runs, list), "The field 'runs' must be a list." contexts = [] for run in runs: @@ -684,7 +701,7 @@ def _runs_to_tab_converter(runs: list | None): return contexts -@custom_fallback_field(get_converter(), {"runs": ("contexts", _runs_to_tab_converter)}) +@fallback_field({"runs": ("contexts", _runs_to_tab_converter)}) @define class Tab(WithFeatures, WithFunctions): """Represents a tab on Dodona.""" diff --git a/tests/exercises/echo/evaluation/input.txt b/tests/exercises/echo/evaluation/input.txt new file mode 100644 index 000000000..b5fc21b37 --- /dev/null +++ b/tests/exercises/echo/evaluation/input.txt @@ -0,0 +1 @@ +Hallo diff --git a/tests/exercises/echo/evaluation/plan-dynamic.yaml b/tests/exercises/echo/evaluation/plan-dynamic.yaml new file mode 100644 index 000000000..7b8a33fe8 --- /dev/null +++ b/tests/exercises/echo/evaluation/plan-dynamic.yaml @@ -0,0 +1,19 @@ +- tab: "Tests" + testcases: + - stdin: + path: "input2.txt" + content: !path input.txt + stdout: "Hallo" + - stdin: + path: "input2.txt" + content: | + Line1 + stdout: "Line1" + - stdin: + path: "input3.txt" + content: "TESTed" + stdout: "TESTed" + - stdin: + path: "input.txt" + content: "This is overwritten." + stdout: "This is overwritten." diff --git a/tests/exercises/echo/evaluation/plan.yaml b/tests/exercises/echo/evaluation/plan.yaml new file mode 100644 index 000000000..48d56381e --- /dev/null +++ b/tests/exercises/echo/evaluation/plan.yaml @@ -0,0 +1,12 @@ +- tab: "Tests" + testcases: + - stdin: "Hallo" + stdout: "Hallo" + - stdin: "Hallo with arguments" + arguments: ["hallo"] + stdout: "Hallo with arguments" + - stdin: + path: "input.txt" + content: !path input.txt + stdout: "Hallo" + diff --git a/tests/test_dsl_legacy.py b/tests/test_dsl_legacy.py new file mode 100644 index 000000000..2ca3f0cba --- /dev/null +++ b/tests/test_dsl_legacy.py @@ -0,0 +1,83 @@ +from tested.dsl.translate_parser import parse_dsl +from tested.serialisation import FunctionCall + + +def test_dsl_tab_legacy_fields(): + # 'unit' is a legacy field for 'tab' + # 'cases' is a legacy field for 'contexts' + yaml_str = """ +- unit: 'Legacy Tab' + cases: + - script: + - expression: 'test()' + return: 5 + """ + suite = parse_dsl(yaml_str) + assert suite.tabs[0].name == "Legacy Tab" + assert len(suite.tabs[0].contexts) == 1 + + +def test_dsl_context_legacy_testcases(): + yaml_str = """ +- tab: 'Test' + contexts: + - testcases: + - expression: 'test()' + return: 5 + """ + suite = parse_dsl(yaml_str) + assert len(suite.tabs[0].contexts[0].testcases) == 1 + + +def test_dsl_suite_legacy_tabs(): + yaml_str = """ +tabs: + - tab: 'Test' + contexts: + - testcases: + - expression: 'test()' + """ + suite = parse_dsl(yaml_str) + assert len(suite.tabs) == 1 + assert suite.tabs[0].name == "Test" + + +def test_dsl_tab_legacy_testcases(): + yaml_str = """ +- tab: 'Test' + testcases: + - expression: 'test()' + """ + suite = parse_dsl(yaml_str) + assert len(suite.tabs[0].contexts) == 1 + tc = suite.tabs[0].contexts[0].testcases[0] + assert isinstance(tc.input, FunctionCall) + assert tc.input.name == "test" + + +def test_dsl_tab_legacy_scripts(): + yaml_str = """ +- unit: 'Test' + scripts: + - expression: 'test()' + """ + suite = parse_dsl(yaml_str) + assert len(suite.tabs[0].contexts) == 1 + tc = suite.tabs[0].contexts[0].testcases[0] + assert isinstance(tc.input, FunctionCall) + assert tc.input.name == "test" + + +def test_dsl_top_level_list_legacy(): + yaml_str = """ +- tab: 'Tab 1' + testcases: + - expression: 'test1()' +- tab: 'Tab 2' + testcases: + - expression: 'test2()' + """ + suite = parse_dsl(yaml_str) + assert len(suite.tabs) == 2 + assert suite.tabs[0].name == "Tab 1" + assert suite.tabs[1].name == "Tab 2" diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index aecf78846..5c82796c8 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -30,6 +30,7 @@ VariableAssignment, ) from tested.testsuite import ( + ContentPath, CustomCheckOracle, FileOutputChannel, FileUrl, @@ -68,10 +69,10 @@ def test_parse_one_tab_ctx(): assert len(context.testcases) == 1 tc = context.testcases[0] assert tc.is_main_testcase() - assert tc.input.stdin.data == "Input string\n" + assert tc.input.stdin.content == "Input string\n" assert tc.input.arguments == ["--arg", "argument"] - assert tc.output.stderr.data == "Error string\n" - assert tc.output.stdout.data == "Output string\n" + assert tc.output.stderr.content == "Error string\n" + assert tc.output.stdout.content == "Output string\n" assert tc.output.exit_code.value == 1 @@ -223,7 +224,7 @@ def test_parse_ctx_with_config(): assert tc3.input.arguments == ["-e"] stdout = tc0.output.stdout - assert stdout.data == "3.34\n" + assert stdout.content == "3.34\n" options = stdout.oracle.options assert len(options) == 3 assert options["tryFloatingPoint"] @@ -231,7 +232,7 @@ def test_parse_ctx_with_config(): assert options["roundTo"] == 2 stdout = tc1.output.stdout - assert stdout.data == "3.337\n" + assert stdout.content == "3.337\n" options = stdout.oracle.options assert len(options) == 3 assert options["tryFloatingPoint"] @@ -239,7 +240,7 @@ def test_parse_ctx_with_config(): assert options["roundTo"] == 3 stdout = tc2.output.stdout - assert stdout.data == "3.3\n" + assert stdout.content == "3.3\n" options = stdout.oracle.options assert len(options) == 3 assert options["tryFloatingPoint"] @@ -247,7 +248,7 @@ def test_parse_ctx_with_config(): assert options["roundTo"] == 2 stderr = tc3.output.stderr - assert stderr.data == " Fail \n" + assert stderr.content == " Fail \n" options = stderr.oracle.options assert len(options) == 2 assert not options["caseInsensitive"] @@ -288,14 +289,14 @@ def test_statements(): assert len(tests0) == 2 assert isinstance(tests0[0].input, VariableAssignment) - assert tests0[0].output.stdout.data == "New safe\n" + assert tests0[0].output.stdout.content == "New safe\n" assert tests0[0].output.stdout.oracle.options["ignoreWhitespace"] assert isinstance(tests0[1].input, FunctionCall) assert tests0[1].output.result.value.data == "Ignore whitespace" assert len(tests1) == 2 assert isinstance(tests1[0].input, VariableAssignment) - assert tests1[0].output.stdout.data == "New safe\n" + assert tests1[0].output.stdout.content == "New safe\n" assert not tests1[0].output.stdout.oracle.options["ignoreWhitespace"] assert isinstance(tests1[1].input, FunctionCall) assert tests1[1].output.result.value.data == 5 @@ -324,7 +325,7 @@ def test_expression_and_main(): assert len(ctx.testcases) == 2 tc = ctx.testcases[0] assert tc.input.arguments == ["-a", "5", "7"] - assert tc.output.stdout.data == "12\n" + assert tc.output.stdout.content == "12\n" assert tc.output.stdout.oracle.options["tryFloatingPoint"] test = ctx.testcases[1] assert isinstance(test.input, FunctionCall) @@ -651,7 +652,7 @@ def test_text_built_in_checks_implied(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.stdout, TextOutputChannel) assert isinstance(test.output.stdout.oracle, GenericTextOracle) - assert test.output.stdout.data == "hallo\n" + assert test.output.stdout.content == "hallo\n" def test_text_built_in_checks_explicit(): @@ -675,7 +676,7 @@ def test_text_built_in_checks_explicit(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.stdout, TextOutputChannel) assert isinstance(test.output.stdout.oracle, GenericTextOracle) - assert test.output.stdout.data == "hallo\n" + assert test.output.stdout.content == "hallo\n" def test_text_custom_checks_correct(): @@ -702,7 +703,7 @@ def test_text_custom_checks_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.stdout, TextOutputChannel) assert isinstance(test.output.stdout.oracle, CustomCheckOracle) - assert test.output.stdout.data == "hallo\n" + assert test.output.stdout.content == "hallo\n" oracle = test.output.stdout.oracle assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") @@ -1211,7 +1212,7 @@ def test_newlines_are_added_to_stdout(): """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) - actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.data + actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.content assert actual_stdout == "12\n" yaml_str2 = """ @@ -1223,7 +1224,7 @@ def test_newlines_are_added_to_stdout(): """ json_str = translate_to_test_suite(yaml_str2) suite = parse_test_suite(json_str) - actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.data + actual_stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout.content assert actual_stdout == "hello\n" @@ -1240,7 +1241,7 @@ def test_newlines_are_added_to_stderr(): """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) - actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data + actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.content assert actual_stderr == "12\n" yaml_str2 = """ @@ -1252,7 +1253,7 @@ def test_newlines_are_added_to_stderr(): """ json_str = translate_to_test_suite(yaml_str2) suite = parse_test_suite(json_str) - actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data + actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.content assert actual_stderr == "hello\n" @@ -1268,7 +1269,7 @@ def test_no_duplicate_newlines_are_added(): """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) - actual = suite.tabs[0].contexts[0].testcases[0].output.stdout.data + actual = suite.tabs[0].contexts[0].testcases[0].output.stdout.content assert actual == "hello\nworld\n" @@ -1286,7 +1287,7 @@ def test_can_disable_normalizing_newlines(): """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) - actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data + actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.content assert actual_stderr == "12" @@ -1300,7 +1301,7 @@ def test_empty_text_data_newlines(): """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) - actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.data + actual_stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr.content assert actual_stderr == "" @@ -1342,3 +1343,60 @@ def test_editor_json_schema_is_valid(): validator = load_schema_validator(file="schema.json") assert isinstance(validator.schema, dict) validator.check_schema(validator.schema) + + +def test_stdin_shorthand_string(): + yaml_str = """ +namespace: "IO" +tabs: + - tab: "Stdin" + testcases: + - stdin: "hello" + stdout: "hello world!\\n" + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + + stdin_def = suite.tabs[0].contexts[0].testcases[0].input.stdin + + assert stdin_def.content == "hello\n" + assert stdin_def.path is None + + +def test_stdin_implicit_file_reference(): + yaml_str = """ +namespace: "IO" +tabs: + - tab: "Stdin" + testcases: + - stdin: + path: "hello.txt" + stdout: "hello world!\\n" +""" + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + + stdin_def = suite.tabs[0].contexts[0].testcases[0].input.stdin + + assert stdin_def.path == "hello.txt" + assert stdin_def.content.path == "hello.txt" + + +def test_stdin_explicit_path_and_content(): + yaml_str = """ +namespace: "IO" +tabs: + - tab: "Stdin" + testcases: + - stdin: + path: "hello.txt" + content: "hello" + stdout: "hello world!\\n" +""" + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + + stdin_def = suite.tabs[0].contexts[0].testcases[0].input.stdin + + assert stdin_def.path == "hello.txt" + assert stdin_def.content == "hello\n" diff --git a/tests/test_functionality.py b/tests/test_functionality.py index da5aeb804..910352a77 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -783,7 +783,7 @@ def test_stdin_and_arguments_use_heredoc(tmp_path: Path, pytestconfig: pytest.Co ) the_input = Testcase( input=MainInput( - arguments=["hello"], stdin=TextData(data="One line\nSecond line\n") + arguments=["hello"], stdin=TextData(content="One line\nSecond line\n") ) ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) @@ -806,7 +806,9 @@ def test_stdin_token_is_unique(tmp_path: Path, pytestconfig: pytest.Config): "top-level-output", ) the_input = Testcase( - input=MainInput(arguments=["hello"], stdin=TextData(data="One line\nSTDIN\n")) + input=MainInput( + arguments=["hello"], stdin=TextData(content="One line\nSTDIN\n") + ) ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) bundle = create_bundle(conf, sys.stdout, suite) diff --git a/tests/test_io_exercises.py b/tests/test_io_exercises.py index ec4a22f53..940f405df 100644 --- a/tests/test_io_exercises.py +++ b/tests/test_io_exercises.py @@ -23,6 +23,28 @@ def test_io_exercise(language: str, tmp_path: Path, pytestconfig: pytest.Config) assert updates.find_status_enum() == ["correct"] +@pytest.mark.parametrize("language", ALL_LANGUAGES) +def test_io_exercise_stdin(language: str, tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, "echo", language, tmp_path, "plan.yaml", "correct" + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] * 3 + + +@pytest.mark.parametrize("language", ALL_LANGUAGES) +def test_io_exercise_stdin_dynamic_file( + language: str, tmp_path: Path, pytestconfig: pytest.Config +): + conf = configuration( + pytestconfig, "echo", language, tmp_path, "plan-dynamic.yaml", "correct" + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] * 4 + + @pytest.mark.parametrize("language", ALL_LANGUAGES) def test_io_exercise_wrong(language: str, tmp_path: Path, pytestconfig: pytest.Config): conf = configuration(pytestconfig, "echo", language, tmp_path, "one.tson", "wrong") diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 7e21ffa02..5ea233573 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -46,7 +46,7 @@ def oracle_config( def test_text_oracle(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config(tmp_path, pytestconfig, {"ignoreWhitespace": False}) - channel = TextOutputChannel(data="expected") + channel = TextOutputChannel(content="expected") result = evaluate_text(config, channel, "expected") assert result.result.enum == Status.CORRECT assert result.readable_expected == "expected" @@ -60,7 +60,7 @@ def test_text_oracle(tmp_path: Path, pytestconfig: pytest.Config): def test_text_oracle_whitespace(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config(tmp_path, pytestconfig, {"ignoreWhitespace": True}) - channel = TextOutputChannel(data="expected") + channel = TextOutputChannel(content="expected") result = evaluate_text(config, channel, "expected ") assert result.result.enum == Status.CORRECT assert result.readable_expected == "expected" @@ -74,7 +74,7 @@ def test_text_oracle_whitespace(tmp_path: Path, pytestconfig: pytest.Config): def test_text_oracle_case_sensitive(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config(tmp_path, pytestconfig, {"caseInsensitive": True}) - channel = TextOutputChannel(data="expected") + channel = TextOutputChannel(content="expected") result = evaluate_text(config, channel, "Expected") assert result.result.enum == Status.CORRECT assert result.readable_expected == "expected" @@ -90,7 +90,7 @@ def test_text_oracle_combination(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config( tmp_path, pytestconfig, {"caseInsensitive": True, "ignoreWhitespace": True} ) - channel = TextOutputChannel(data="expected") + channel = TextOutputChannel(content="expected") result = evaluate_text(config, channel, "Expected ") assert result.result.enum == Status.CORRECT assert result.readable_expected == "expected" @@ -106,7 +106,7 @@ def test_text_oracle_rounding(tmp_path: Path, pytestconfig: pytest.Config): config = oracle_config( tmp_path, pytestconfig, {"tryFloatingPoint": True, "applyRounding": True} ) - channel = TextOutputChannel(data="1.333") + channel = TextOutputChannel(content="1.333") result = evaluate_text(config, channel, "1.3333333") assert result.result.enum == Status.CORRECT assert result.readable_expected == "1.333" @@ -124,7 +124,7 @@ def test_text_oracle_round_to(tmp_path: Path, pytestconfig: pytest.Config): pytestconfig, {"tryFloatingPoint": True, "applyRounding": True, "roundTo": 1}, ) - channel = TextOutputChannel(data="1.3") + channel = TextOutputChannel(content="1.3") result = evaluate_text(config, channel, "1.3333333") assert result.result.enum == Status.CORRECT assert result.readable_expected == "1.3" diff --git a/tests/test_parsing.py b/tests/test_parsing.py new file mode 100644 index 000000000..5729b37b3 --- /dev/null +++ b/tests/test_parsing.py @@ -0,0 +1,246 @@ +from decimal import Decimal + +import pytest +from attrs import define + +from tested.parsing import fallback_field, get_converter, ignore_field + + +@define +class DummyClass: + field1: str + field2: int = 0 + + +def test_ignore_field(): + @ignore_field("to_ignore") + @define + class IgnoreDummy: + field1: str + field2: int = 0 + + converter = get_converter() + data = {"field1": "value", "field2": 10, "to_ignore": "some_value"} + result = converter.structure(data, IgnoreDummy) + assert result.field1 == "value" + assert result.field2 == 10 + + +def test_ignore_multiple_fields(): + @ignore_field("ignore1", "ignore2") + @define + class IgnoreMultipleDummy: + field1: str + + converter = get_converter() + data = {"field1": "value", "ignore1": 1, "ignore2": 2} + result = converter.structure(data, IgnoreMultipleDummy) + assert result.field1 == "value" + + +def test_fallback_field(): + @fallback_field({"old_field": "new_field"}) + @define + class FallbackDummy: + new_field: str + + converter = get_converter() + data = {"old_field": "value"} + result = converter.structure(data, FallbackDummy) + assert result.new_field == "value" + + data = {"old_field": "old", "new_field": "new"} + result = converter.structure(data, FallbackDummy) + assert result.new_field == "new" + + +def test_fallback_field_custom(): + def mapper(old_val, _d): + return f"mapped_{old_val}" + + @fallback_field({"old": ("new", mapper)}) + @define + class CustomFallbackDummy: + new: str + + converter = get_converter() + data = {"old": "value"} + result = converter.structure(data, CustomFallbackDummy) + assert result.new == "mapped_value" + + data = {"old": "old", "new": "new"} + with pytest.raises(ValueError) as exc_info: + converter.structure(data, CustomFallbackDummy) + assert "You cannot use new and old simultaneously" in str(exc_info.value) + + +def test_fallback_field_custom_with_context(): + def mapper(old_val, d): + return f"{old_val}_{d.get('other', 'none')}" + + @fallback_field({"old": ("new", mapper)}) + @define + class CustomFallbackContextDummy: + new: str + other: str = "default" + + converter = get_converter() + data = {"old": "value", "other": "context"} + result = converter.structure(data, CustomFallbackContextDummy) + assert result.new == "value_context" + + +def test_decorator_stacking(): + @fallback_field( + { + "old_custom": ("field1", lambda x, _: f"custom_{x}"), + "old_fallback": "field2", + } + ) + @ignore_field("to_ignore") + @define + class StackedDummy: + field1: str + field2: int + field3: str = "default" + + converter = get_converter() + data = { + "old_custom": "val1", + "old_fallback": 42, + "to_ignore": "hidden", + "field3": "provided", + } + result = converter.structure(data, StackedDummy) + assert result.field1 == "custom_val1" + assert result.field2 == 42 + assert result.field3 == "provided" + + +def test_non_dict_input(): + @ignore_field("field") + @define + class NonDictDummy: + field: str + + converter = get_converter() + # If input is not a dict, it should pass through to previous hook (which might fail or handle it) + # Actually, cattrs' structure for a class usually expects a dict if it's using make_dict_structure_fn + with pytest.raises(Exception): + converter.structure("not a dict", NonDictDummy) + + +def test_chain_structure_hook_registry_isolation(): + # Verify that decorators on one class don't affect another + @ignore_field("extra") + @define + class ClassA: + name: str + + @define + class ClassB: + name: str + extra: str + + converter = get_converter() + + # ClassA should ignore "extra" + data_a = {"name": "A", "extra": "ignored"} + result_a = converter.structure(data_a, ClassA) + assert result_a.name == "A" + + # ClassB should NOT ignore "extra" (it's a required field here) + data_b = {"name": "B", "extra": "present"} + result_b = converter.structure(data_b, ClassB) + assert result_b.name == "B" + assert result_b.extra == "present" + + +def test_decimal_hooks(): + from tested.parsing import structure_decimal, unstructure_decimal + + d = Decimal("1.23") + assert unstructure_decimal(d) == "1.23" + assert structure_decimal("1.23", Decimal) == d + assert structure_decimal(1.23, Decimal) == d + + +def test_structure_every_union(): + from typing import Union + + from tested.parsing import structure_every_union + from tested.serialisation import Identifier + + # Test Union[int, str] + the_type = Union[int, str] + assert structure_every_union(1, the_type) == 1 + assert structure_every_union("a", the_type) == "a" + + # Test Union[None, int] + the_type = Union[None, int] + assert structure_every_union(None, the_type) is None + assert structure_every_union(5, the_type) == 5 + + # Test Union[bool, int] + the_type = Union[bool, int] + assert structure_every_union(True, the_type) is True + assert structure_every_union(0, the_type) == 0 + + # Test Identifier in Union + the_type = Union[Identifier, int] + result = structure_every_union("my_id", the_type) + assert isinstance(result, Identifier) + assert result == "my_id" + + # Test failure + with pytest.raises(TypeError): + # We need something that doesn't match any branch in structure_every_union + # The branches are: + # 1. None + # 2. bool + # 3. int + # 4. float + # 5. Identifier (str) + # 6. _suite_converter.structure (generic fallback) + + # If we use a list and the type is Union[int, str], it might fail in cattrs + # but structure_every_union raises TypeError if it falls through. + # However, cattrs might raise an exception inside the loop, which is caught and rejected. + structure_every_union([], Union[int, Decimal]) + + +def test_initialise_converter_idempotent(): + from tested.parsing import initialise_converter + + # Should not raise any errors when called multiple times + initialise_converter() + initialise_converter() + + +def test_parse_json_value(): + from tested.datatypes import BasicStringTypes + from tested.parsing import parse_json_value + from tested.serialisation import StringType + + json_str = '{"type": "text", "data": "hello"}' + result = parse_json_value(json_str) + assert isinstance(result, StringType) + assert result.type == BasicStringTypes.TEXT + assert result.data == "hello" + + +def test_parse_json_suite_and_to_json(): + from tested.parsing import parse_json_suite, suite_to_json + from tested.testsuite import Suite + + # A very minimal suite + json_str = '{"tabs": [], "namespace": "test_namespace"}' + suite = parse_json_suite(json_str) + assert isinstance(suite, Suite) + assert suite.tabs == [] + assert suite.namespace == "test_namespace" + + # Convert back to json + back_to_json = suite_to_json(suite) + # The default Suite might have other fields, but namespace should be there if not default + assert "test_namespace" in back_to_json diff --git a/tests/test_suite.py b/tests/test_suite.py index d194691bb..a9dd4100e 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -89,7 +89,7 @@ def test_input_deprecated_attribute_is_accepted(): """ result = get_converter().loads(scheme, MainInput) assert isinstance(result.stdin, TextData) - assert result.stdin.data == "input-1" + assert result.stdin.content == "input-1" def test_text_show_expected_is_accepted(): @@ -101,7 +101,7 @@ def test_text_show_expected_is_accepted(): } """ result = get_converter().loads(scheme, TextOutputChannel) - assert result.data == "hallo" + assert result.content == "hallo" def test_file_show_expected_is_accepted(): diff --git a/tests/test_testsuite_legacy.py b/tests/test_testsuite_legacy.py new file mode 100644 index 000000000..bed2ebc7b --- /dev/null +++ b/tests/test_testsuite_legacy.py @@ -0,0 +1,194 @@ +import pytest +from attrs import define + +from tested.parsing import fallback_field, get_converter, ignore_field, parse_json_suite +from tested.serialisation import Identifier, StringType +from tested.testsuite import ( + ContentPath, + Context, + ExceptionBuiltin, + ExceptionOutputChannel, + FileOutputChannel, + GenericExceptionOracle, + GenericTextOracle, + GenericValueOracle, + LanguageSpecificOracle, + MainInput, + SupportedLanguage, + Tab, + Testcase, + TextBuiltin, + TextData, + TextOutputChannel, + ValueBuiltin, + ValueOutputChannel, +) + + +def test_language_specific_oracle_legacy_evaluators(): + converter = get_converter() + data = { + "evaluators": {"python": {"file": "test.py", "name": "test"}}, + "arguments": {"python": ["arg1"]}, + } + result = converter.structure(data, LanguageSpecificOracle) + assert "python" in result.functions + assert result.functions[SupportedLanguage.PYTHON].name == "test" + assert result.arguments[SupportedLanguage.PYTHON] == ["arg1"] + + +def test_text_data_legacy_data_string(): + data = {"data": "some content"} + result = get_converter().structure(data, TextData) + assert result.content == "some content" + + +def test_text_data_legacy_data_file(): + data = {"data": "path/to/file.txt", "type": "file"} + result = get_converter().structure(data, TextData) + + assert isinstance(result.content, ContentPath) + assert result.content.path == "path/to/file.txt" + + +def test_text_output_channel_legacy_evaluator(): + data = {"data": "expected", "evaluator": {"name": "text"}} + result = get_converter().structure(data, TextOutputChannel) + assert result.content == "expected" + assert isinstance(result.oracle, GenericTextOracle) + assert result.oracle.name == TextBuiltin.TEXT + + +def test_file_output_channel_legacy_evaluator(): + data = { + "expected_path": "exp.txt", + "actual_path": "act.txt", + "evaluator": {"name": "file"}, + } + result = get_converter().structure(data, FileOutputChannel) + assert result.expected_path == "exp.txt" + assert result.actual_path == "act.txt" + assert isinstance(result.oracle, GenericTextOracle) + assert result.oracle.name == TextBuiltin.FILE + + +def test_value_output_channel_legacy_evaluator(): + data = {"value": {"type": "text", "data": "val"}, "evaluator": {"name": "value"}} + result = get_converter().structure(data, ValueOutputChannel) + assert isinstance(result.value, StringType) + assert isinstance(result.oracle, GenericValueOracle) + assert result.oracle.name == ValueBuiltin.VALUE + + +def test_exception_output_channel_legacy_evaluator(): + data = {"exception": {"message": "error"}, "evaluator": {"name": "exception"}} + result = get_converter().structure(data, ExceptionOutputChannel) + assert result.exception is not None + assert result.exception.message == "error" + assert isinstance(result.oracle, GenericExceptionOracle) + assert result.oracle.name == ExceptionBuiltin.EXCEPTION + + +def test_tab_legacy_runs(): + data = { + "name": "Tab 1", + "runs": [ + {"run": {"input": {"arguments": ["arg1"], "main_call": True}}}, + {"contexts": [{"testcases": [{"input": "5"}]}]}, + ], + } + result = get_converter().structure(data, Tab) + assert len(result.contexts) == 2 + assert isinstance(result.contexts[0].testcases[0].input, MainInput) + assert result.contexts[0].testcases[0].input.arguments == ["arg1"] + + assert isinstance(result.contexts[1].testcases[0].input, Identifier) + assert len(result.contexts[1].testcases[0].input) == 1 + assert result.contexts[1].testcases[0].input == "5" + + +def test_ignore_fields(): + converter = get_converter() + + # TextData ignores 'type' (used in converter, but should be popped) + data_text = {"content": "text", "type": "file"} + result_text = converter.structure(data_text, TextData) + assert result_text.content == "text" + + # Testcase ignores 'essential' + data_testcase = {"input": {"arguments": [], "main_call": True}, "essential": True} + result_testcase = converter.structure(data_testcase, Testcase) + assert isinstance(result_testcase.input, MainInput) + + # Context ignores 'link_files' + data_context = {"testcases": [], "link_files": []} + result_context = converter.structure(data_context, Context) + assert result_context.testcases == [] + + +def test_full_suite_legacy_format(): + json_suite = """ + { + "namespace": "test", + "tabs": [ + { + "name": "Tab 1", + "runs": [ + { + "run": { + "input": { + "arguments": ["a"], + "main_call": true + }, + "output": { + "stdout": { + "data": "out", + "evaluator": {"name": "text"} + } + } + } + } + ] + } + ] + } + """ + suite = parse_json_suite(json_suite) + assert suite.namespace == "test" + assert len(suite.tabs) == 1 + assert suite.tabs[0].name == "Tab 1" + assert len(suite.tabs[0].contexts) == 1 + tc = suite.tabs[0].contexts[0].testcases[0] + assert isinstance(tc.input, MainInput) + assert isinstance(tc.output.stdout, TextOutputChannel) + assert tc.output.stdout.content == "out" + assert isinstance(tc.output.stdout.oracle, GenericTextOracle) + + +@fallback_field({"old": "new"}) +@ignore_field("old") +@define +class OrderTest: + new: str + + +def test_decorator_order(): + # If fallback runs first, "old" is mapped to "new", then "old" is ignored. + # If ignore runs first, "old" is removed, then fallback sees nothing. + data = {"old": "value"} + result = get_converter().structure(data, OrderTest) + assert result.new == "value" + + +@ignore_field("old") +@fallback_field({"old": "new"}) +@define +class ReverseOrderTest: + new: str + + +def test_reverse_decorator_order(): + data = {"old": "value"} + # If ignore runs first, this should fail because "new" is missing. + with pytest.raises(Exception): + get_converter().structure(data, ReverseOrderTest) diff --git a/tests/tested-draft7.json b/tests/tested-draft7.json index 1e49ec747..0f16450e0 100644 --- a/tests/tested-draft7.json +++ b/tests/tested-draft7.json @@ -28,7 +28,8 @@ "object", "string", "oracle", - "expression" + "expression", + "path" ] }, "stringArray": {