Skip to content

Add support for output_files #618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
niknetniko merged 24 commits into from
Mar 5, 2026
Merged

Add support for output_files #618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
b12e29e
Implement file oracles
niknetniko Feb 26, 2026
48635da
Update DSL
niknetniko Feb 26, 2026
36d2ea9
Linting and typing
niknetniko Feb 26, 2026
9cb191f
Add some utilities
niknetniko Feb 28, 2026
b33d61d
Always use filename as channel
niknetniko Feb 28, 2026
2d7c580
Take files into account in execution planning
niknetniko Feb 28, 2026
f532d78
Do not add message if no files
niknetniko Feb 28, 2026
a14b75c
Add some tests
niknetniko Feb 28, 2026
68bc9d3
Add more tests
niknetniko Feb 28, 2026
2700b68
Tweak test
niknetniko Feb 28, 2026
fefe6b1
Test and fix custom checks with files
niknetniko Feb 28, 2026
355c9ab
Better deprecation messages
niknetniko Mar 5, 2026
5817cb7
Pass index
niknetniko Mar 5, 2026
a2b7294
Improve error message
niknetniko Mar 5, 2026
c0ab7d8
Remove unused import
niknetniko Mar 5, 2026
125f8ad
Fix typo
niknetniko Mar 5, 2026
aebc0bc
Fix wrong assert
niknetniko Mar 5, 2026
de3615a
Fix typo
niknetniko Mar 5, 2026
915748c
Temp
niknetniko Mar 5, 2026
027e30d
Fix potential future bug
niknetniko Mar 5, 2026
ec3beb0
Merge branch 'files-features' into output-files
niknetniko Mar 5, 2026
b05a68b
Do some more tweaks
niknetniko Mar 5, 2026
dd0657c
Remove misleading comment
niknetniko Mar 5, 2026
4cf4b28
Add some more tests and be stricter
niknetniko Mar 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions tested/dsl/schema-strict.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@
]
},
"files" : {
"description" : "Deprecated: use input_files instead.",
"type" : "array",
"items" : {
"$ref" : "#/definitions/file"
Expand All @@ -367,6 +368,10 @@
"$ref" : "#/definitions/textOutputChannel"
},
"file": {
"description" : "Deprecated: use output_files instead.",
"$ref" : "#/definitions/fileOutputChannel"
},
"output_files": {
"description" : "Expected files generated by the submission.",
"$ref" : "#/definitions/fileOutputChannel"
},
Expand Down Expand Up @@ -611,6 +616,35 @@
}
}
},
{
"type" : "object",
"description" : "Built-in oracle for files.",
"required" : [
"data"
],
"properties" : {
"data" : {
"type" : "array",
"description" : "Files to expect.",
"items" : {
"$ref" : "#/definitions/fileDataFullyRequired"
}
},
"oracle" : {
"const" : "builtin"
},
"config" : {
"$ref" : "#/definitions/fileConfigurationOptions"
}
}
},
{
"type" : "array",
"description" : "Built-in oracle for files.",
"items" : {
"$ref" : "#/definitions/fileDataFullyRequired"
}
},
{
"type" : "object",
"description" : "Custom oracle for file values.",
Expand Down Expand Up @@ -656,6 +690,49 @@
}
}
}
},
{
"type" : "object",
"description" : "Custom oracle for file values.",
"required" : [
"oracle",
"data"
],
"properties" : {
"oracle" : {
"const" : "custom_check"
},
"data" : {
"type" : "array",
"description" : "Files to expect.",
"items" : {
"$ref" : "#/definitions/fileDataFullyRequired"
}
},
"file" : {
"type" : "string",
"description" : "The path to the file containing the custom check function."
},
"name" : {
"type" : "string",
"description" : "The name of the custom check function.",
"default" : "evaluate"
},
"arguments" : {
"type" : "array",
"description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
"items" : {
"$ref" : "#/definitions/yamlValueOrPythonExpression"
}
},
"languages": {
"type" : "array",
"description" : "Which programming languages are supported by this oracle.",
"items" : {
"$ref" : "#/definitions/programmingLanguage"
}
}
}
}
]
},
Expand Down Expand Up @@ -952,6 +1029,27 @@
"description" : "Path to the file, relative to the workdir. Used to display in the output."
}
}
},
"fileDataFullyRequired": {
"type": "object",
"additionalProperties" : false,
"required" : [
"path",
"content"
],
"properties": {
"content": {
"type": [
"string",
"path"
],
"description" : "Content of the file, which will be provided inline or written to disk in the workdir. If a !path, the file contents will be read from the provided path."
},
"path": {
"type": "string",
"description" : "Path to the file, relative to the workdir. Used to display in the output."
}
}
}
}
}
99 changes: 99 additions & 0 deletions tested/dsl/schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@
]
},
"files" : {
"description" : "Deprecated: use input_files instead.",
"type" : "array",
"items" : {
"$ref" : "#/definitions/file"
Expand All @@ -366,6 +367,14 @@
"description" : "Expected output at stdout",
"$ref" : "#/definitions/textOutputChannel"
},
"file": {
"description" : "Deprecated: use output_files instead.",
"$ref" : "#/definitions/fileOutputChannel"
},
"output_files": {
"description" : "Expected files generated by the submission.",
"$ref" : "#/definitions/fileOutputChannel"
},
"exit_code" : {
"type" : "integer",
"description" : "Expected exit code for the run"
Expand Down Expand Up @@ -611,6 +620,35 @@
}
}
},
{
"type" : "object",
"description" : "Built-in oracle for files.",
"required" : [
"data"
],
"properties" : {
"data" : {
"type" : "array",
"description" : "Files to expect.",
"items" : {
"$ref" : "#/definitions/fileDataFullyRequired"
}
},
"oracle" : {
"const" : "builtin"
},
"config" : {
"$ref" : "#/definitions/fileConfigurationOptions"
}
}
},
{
"type" : "array",
"description" : "Built-in oracle for files.",
"items" : {
"$ref" : "#/definitions/fileDataFullyRequired"
}
},
{
"type" : "object",
"description" : "Custom oracle for file values.",
Expand Down Expand Up @@ -656,6 +694,49 @@
}
}
}
},
{
"type" : "object",
"description" : "Custom oracle for file values.",
"required" : [
"oracle",
"data"
],
"properties" : {
"oracle" : {
"const" : "custom_check"
},
"data" : {
"type" : "array",
"description" : "Files to expect.",
"items" : {
"$ref" : "#/definitions/fileDataFullyRequired"
}
},
"file" : {
"type" : "string",
"description" : "The path to the file containing the custom check function."
},
"name" : {
"type" : "string",
"description" : "The name of the custom check function.",
"default" : "evaluate"
},
"arguments" : {
"type" : "array",
"description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
"items" : {
"$ref" : "#/definitions/yamlValueOrPythonExpression"
}
},
"languages": {
"type" : "array",
"description" : "Which programming languages are supported by this oracle.",
"items" : {
"$ref" : "#/definitions/programmingLanguage"
}
}
}
}
]
},
Expand Down Expand Up @@ -940,6 +1021,24 @@
"description" : "Path to the file, relative to the workdir. Used to display in the output."
}
}
},
"fileDataFullyRequired": {
"type": "object",
"additionalProperties" : false,
"required" : [
"path",
"content"
],
"properties": {
"content": {
"type": "string",
"description" : "Content of the file, which will be provided inline or written to disk in the workdir. If a !path, the file contents will be read from the provided path."
},
"path": {
"type": "string",
"description" : "Path to the file, relative to the workdir. Used to display in the output."
}
}
}
}
}
72 changes: 47 additions & 25 deletions tested/dsl/translate_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,10 +305,12 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext":
return evolve(self, files=the_files, config=the_config)

def merge_inheritable_with_specific_config(
self, level: YamlDict, config_name: str
self, level: YamlObject, config_name: str
) -> dict:
inherited_options = self.config.get(config_name, dict())
specific_options = level.get("config", dict())
specific_options = (
level.get("config", dict()) if isinstance(level, dict) else dict()
)
assert isinstance(
specific_options, dict
), f"The config options for {config_name} must be a dictionary, not a {type(specific_options)}"
Expand Down Expand Up @@ -515,32 +517,50 @@ def _convert_text_output_channel(
def _convert_file_output_channel(
stream: YamlObject, context: DslContext, config_name: str
) -> FileOutputChannel:
assert isinstance(stream, dict)

expected = str(stream["content"])
actual = str(stream["location"])
config = context.merge_inheritable_with_specific_config(stream, config_name)
if "mode" not in config:
config["mode"] = "full"
assert config["mode"] in (
"full",
"line",
), f"The file oracle only supports modes full and line, not {config['mode']}"

if isinstance(stream, list):
# We have a list of files. This is not supported in the old way, so nothing special.
files = [_convert_text_data_required_path(f) for f in stream]
oracle = GenericTextOracle(name=TextBuiltin.FILE, options=config)
elif isinstance(stream, dict):
if "content" in stream:
# Handle legacy stuff.
expected = str(stream["content"])
actual = str(stream["location"])
files = [
TextData(
path=actual,
content=ContentPath(path=expected),
)
]
else:
file_list = stream.get("data")
assert isinstance(
file_list, list
), "The data key must be a list of expected files."
files = [_convert_text_data_required_path(f) for f in file_list]

if "oracle" not in stream or stream["oracle"] == "builtin":
config = context.merge_inheritable_with_specific_config(stream, config_name)
if "mode" not in config:
config["mode"] = "full"

assert config["mode"] in (
"full",
"line",
), f"The file oracle only supports modes full and line, not {config['mode']}"
return FileOutputChannel(
expected_path=expected,
actual_path=actual,
oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config),
)
elif stream["oracle"] == "custom_check":
return FileOutputChannel(
expected_path=expected,
actual_path=actual,
oracle=_convert_custom_check_oracle(stream),
if "oracle" not in stream or stream["oracle"] == "builtin":
oracle = GenericTextOracle(name=TextBuiltin.FILE, options=config)
elif stream["oracle"] == "custom_check":
oracle = _convert_custom_check_oracle(stream)
else:
raise TypeError(f"Unknown file oracle type: {stream['oracle']}")
else:
raise InvalidDslError(
f"Invalid file output channel: {stream}.\n\n"
"File output channels must be a list of files, or a dictionary with a 'data' key."
)
raise TypeError(f"Unknown file oracle type: {stream['oracle']}")

return FileOutputChannel(files=files, oracle=oracle)


def _convert_yaml_value(stream: YamlObject) -> Value | None:
Expand Down Expand Up @@ -694,6 +714,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
output.stdout = _convert_text_output_channel(stdout, context, "stdout")
if (file := testcase.get("file")) is not None:
output.file = _convert_file_output_channel(file, context, "file")
if (file := testcase.get("output_files")) is not None:
output.file = _convert_file_output_channel(file, context, "output_files")
if (stderr := testcase.get("stderr")) is not None:
output.stderr = _convert_text_output_channel(stderr, context, "stderr")
if (exception := testcase.get("exception")) is not None:
Expand Down
Loading