Merge pull request #608 from dodona-edu/enhancement/stdin

Improve file handling for stdin

Author: niknetniko

tested/configs.py

@@ -67,7 +67,7 @@ class Options:
     """
 
 
-@fallback_field(get_converter(), {"testplan": "test_suite", "plan_name": "test_suite"})
+@fallback_field({"testplan": "test_suite", "plan_name": "test_suite"})
 @define
 class DodonaConfig:
     resources: Path

tested/dsl/schema-strict.json

@@ -276,11 +276,18 @@
         },
         "stdin" : {
           "description" : "Stdin for this context",
-          "type" : [
-            "string",
-            "number",
-            "integer",
-            "boolean"
+          "oneOf" : [
+            {
+              "type" : [
+                "string",
+                "number",
+                "integer",
+                "boolean"
+              ]
+            },
+            {
+              "$ref": "#/definitions/fileData"
+            }
           ]
         },
         "arguments" : {
@@ -890,6 +897,35 @@
           "$ref" : "#/definitions/fileConfigurationOptions"
         }
       }
+    },
+    "fileData": {
+      "type": "object",
+      "additionalProperties" : false,
+      "anyOf" : [
+        {
+          "required" : [
+            "content"
+          ]
+        },
+        {
+          "required" : [
+            "path"
+          ]
+        }
+      ],
+      "properties": {
+        "content": {
+          "type": [
+            "string",
+            "path"
+          ],
+          "description" : "Content of the file, which will be provided inline or written to disk in the workdir. If a !path, the file contents will be read from the provided path."
+        },
+        "path": {
+          "type": "string",
+          "description" : "Path to the file, relative to the workdir. Used to display in the output."
+        }
+      }
     }
   }
 }

tested/dsl/schema.json

@@ -276,11 +276,18 @@
         },
         "stdin" : {
           "description" : "Stdin for this context",
-          "type" : [
-            "string",
-            "number",
-            "integer",
-            "boolean"
+          "oneOf" : [
+            {
+              "type" : [
+                "string",
+                "number",
+                "integer",
+                "boolean"
+              ]
+            },
+            {
+              "$ref": "#/definitions/fileData"
+            }
           ]
         },
         "arguments" : {
@@ -884,6 +891,32 @@
           "$ref" : "#/definitions/fileConfigurationOptions"
         }
       }
+    },
+    "fileData": {
+      "type": "object",
+      "additionalProperties" : false,
+      "anyOf" : [
+        {
+          "required" : [
+            "content"
+          ]
+        },
+        {
+          "required" : [
+            "path"
+          ]
+        }
+      ],
+      "properties": {
+        "content": {
+          "type": "string",
+          "description" : "Content of the file, which will be provided inline or written to disk in the workdir. If a !path, the file contents will be read from the provided path."
+        },
+        "path": {
+          "type": "string",
+          "description" : "Path to the file, relative to the workdir. Used to display in the output."
+        }
+      }
     }
   }
 }

tested/dsl/translate_parser.py

@@ -45,6 +45,7 @@
     Value,
 )
 from tested.testsuite import (
+    ContentPath,
     Context,
     CustomCheckOracle,
     EmptyChannel,
@@ -88,9 +89,22 @@ class ReturnOracle(dict):
     pass
 
 
+class ContentPathString(str):
+    pass
+
+
 OptionDict = dict[str, int | bool]
 YamlObject = (
-    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle
+    YamlDict
+    | list
+    | bool
+    | float
+    | int
+    | str
+    | None
+    | ExpressionString
+    | ReturnOracle
+    | ContentPathString
 )
 
 
@@ -138,6 +152,12 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     return ReturnOracle(result)
 
 
+def _return_path(loader: yaml.Loader, node: yaml.Node) -> ContentPathString:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(result, str), f"A path must be a string, got {result}"
+    return ContentPathString(result)
+
+
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
     Parse a string or stream to YAML.
@@ -148,6 +168,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
             yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
     yaml.add_constructor("!expression", _expression_string, loader)
     yaml.add_constructor("!oracle", _return_oracle, loader)
+    yaml.add_constructor("!path", _return_path, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
@@ -187,6 +208,10 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
 
+def is_path(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, ContentPathString)
+
+
 def load_schema_validator(
     dsl_object: YamlObject = None, file: str = "schema-strict.json"
 ) -> Validator:
@@ -215,9 +240,13 @@ def validate_tested_dsl_expression(value: object) -> bool:
         schema_object = json.load(schema_file)
 
     original_validator: Type[Validator] = validator_for(schema_object)
-    type_checker = original_validator.TYPE_CHECKER.redefine(
-        "oracle", is_oracle
-    ).redefine("expression", is_expression)
+    type_checker = original_validator.TYPE_CHECKER.redefine_many(
+        {
+            "oracle": is_oracle,
+            "expression": is_expression,
+            "path": is_path,
+        }
+    )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(
         validate_tested_dsl_expression
@@ -469,16 +498,16 @@ def _convert_text_output_channel(
         data = raw_data
 
     if isinstance(stream, str):
-        return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config))
+        return TextOutputChannel(content=data, oracle=GenericTextOracle(options=config))
     else:
         assert isinstance(stream, dict)
         if "oracle" not in stream or stream["oracle"] == "builtin":
             return TextOutputChannel(
-                data=data, oracle=GenericTextOracle(options=config)
+                content=data, oracle=GenericTextOracle(options=config)
             )
         elif stream["oracle"] == "custom_check":
             return TextOutputChannel(
-                data=data, oracle=_convert_custom_check_oracle(stream)
+                content=data, oracle=_convert_custom_check_oracle(stream)
             )
         raise TypeError(f"Unknown text oracle type: {stream['oracle']}")
 
@@ -599,8 +628,33 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
         return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None
     else:
         if "stdin" in testcase:
-            assert isinstance(testcase["stdin"], str)
-            stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"]))
+            if isinstance(testcase["stdin"], dict):
+                stdin_object = testcase["stdin"]
+                raw_content = stdin_object.get("content")
+                raw_path = stdin_object.get("path")
+
+                if isinstance(raw_content, ContentPathString):
+                    file_content = ContentPath(path=raw_content)
+                elif isinstance(raw_content, str):
+                    file_content = _ensure_trailing_newline(raw_content)
+                elif isinstance(raw_path, str):
+                    file_content = ContentPath(path=raw_path)
+                else:
+                    assert (
+                        False
+                    ), f"Invalid stdin content is required but got {type(raw_content)}"
+
+                assert raw_path is None or isinstance(
+                    raw_path, str
+                ), "Path must be a string if given."
+
+                file_path = raw_path
+            else:
+                assert isinstance(testcase["stdin"], str)
+                file_content = _ensure_trailing_newline(testcase["stdin"])
+                file_path = None
+
+            stdin = TextData(content=file_content, path=file_path)
         else:
             stdin = EmptyChannel.NONE
         arguments = testcase.get("arguments", [])

tested/judge/execution.py

@@ -1,5 +1,6 @@
 import itertools
 import logging
+import shutil
 from pathlib import Path
 
 from attrs import define
@@ -16,6 +17,7 @@
 )
 from tested.languages.conventionalize import selector_name
 from tested.languages.preparation import exception_file, value_file
+from tested.testsuite import ContentPath
 from tested.utils import safe_del
 
 _logger = logging.getLogger(__name__)
@@ -180,6 +182,26 @@ def set_up_unit(
         # See https://github.com/dodona-edu/universal-judge/issues/57
         destination.hardlink_to(origin)
 
+    # Create dynamically generated files if necessary.
+    dynamically_generated_file = unit.get_dynamically_generated_files()
+    if dynamically_generated_file is not None:
+        destination = execution_dir / dynamically_generated_file.path
+
+        if isinstance(dynamically_generated_file.content, ContentPath):
+            _logger.debug(
+                f"Copying input file %s to %s",
+                dynamically_generated_file.content.path,
+                destination,
+            )
+            source_file = (
+                bundle.config.resources / dynamically_generated_file.content.path
+            )
+            shutil.copy2(source_file, destination)
+        else:
+            _logger.debug(f"Creating dynamically generated file %s", destination)
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            destination.write_text(dynamically_generated_file.content)
+
     return execution_dir, dependencies
 
 

tested/judge/planning.py

@@ -13,7 +13,7 @@
 from tested.dodona import AnnotateCode, Message, Status
 from tested.languages.conventionalize import execution_name
 from tested.languages.language import FileFilter
-from tested.testsuite import Context, EmptyChannel, MainInput
+from tested.testsuite import ContentPath, Context, EmptyChannel, MainInput
 
 
 @define
@@ -33,6 +33,12 @@ class PlannedContext:
     context_index: int
 
 
+@define(frozen=True)
+class DynamicallyGeneratedFile:
+    path: str
+    content: ContentPath | str
+
+
 @define
 class PlannedExecutionUnit:
     """
@@ -56,6 +62,27 @@ def has_main_testcase(self) -> bool:
     def has_exit_testcase(self) -> bool:
         return self.contexts[-1].context.has_exit_testcase()
 
+    def get_dynamically_generated_files(self) -> DynamicallyGeneratedFile | None:
+        if not self.has_main_testcase():
+            return None
+
+        stdin = cast(MainInput, self.contexts[0].context.testcases[0].input).stdin
+
+        if stdin == EmptyChannel.NONE:
+            return None
+
+        if not stdin.is_dynamically_generated():
+            return None
+
+        # For type checking, in the future should be removable.
+        if stdin.path is None:
+            return None
+
+        return DynamicallyGeneratedFile(
+            path=stdin.path,
+            content=stdin.content,
+        )
+
 
 @define
 class ExecutionPlan:

tested/languages/generation.py

@@ -128,12 +128,16 @@ def get_readable_input(
         args = f"$ {command}"
         # Determine the stdin
         if isinstance(case.input.stdin, TextData):
-            stdin = case.input.stdin.get_data_as_string(bundle.config.resources)
+            if case.input.stdin.path is not None:
+                stdin = Path(case.input.stdin.path)
+            else:
+                stdin = case.input.stdin.get_data_as_string(bundle.config.resources)
         else:
             stdin = ""
 
-        # If we have both stdin and arguments, we use a here-document.
-        if case.input.arguments and stdin:
+        if stdin and isinstance(stdin, Path):
+            text = f"${args} < {stdin}"
+        elif case.input.arguments and stdin:
             assert stdin[-1] == "\n", "stdin must end with a newline"
             delimiter = _get_heredoc_token(stdin)
             text = f"{args} << '{delimiter}'\n{stdin}{delimiter}"

tested/languages/preparation.py

@@ -487,10 +487,10 @@ def prepare_context(
     """
     language = bundle.config.programming_language
     resources = bundle.config.resources
-    before_code = context.before.get(language, TextData(data="")).get_data_as_string(
+    before_code = context.before.get(language, TextData(content="")).get_data_as_string(
         resources
     )
-    after_code = context.after.get(language, TextData(data="")).get_data_as_string(
+    after_code = context.after.get(language, TextData(content="")).get_data_as_string(
         resources
     )
     testcases, evaluator_names = prepare_testcases(bundle, context)

tested/oracles/common.py

@@ -74,7 +74,6 @@ class OracleResult:
 
 
 @fallback_field(
-    get_converter(),
     {
         "readableExpected": "readable_expected",
         "readableActual": "readable_actual",