Add some more tests and be stricter

Author: niknetniko

tested/judge/evaluation.py

@@ -498,8 +498,9 @@ def _add_channel(
     if should_show(output, channel):
         if isinstance(output, FileOutputChannel):
             for i, file in enumerate(output.files):
-                if file.path is None:
-                    continue
+                assert (
+                    file.path is not None
+                ), "File path must be set when using output_files"
 
                 updates.append(
                     StartTest(

tested/testsuite.py

@@ -316,6 +316,11 @@ class FileOutputChannel(WithFeatures):
         factory=lambda: GenericTextOracle(name=TextBuiltin.FILE)
     )
 
+    def __attrs_post_init__(self):
+        for f in self.files:
+            if f.path is None:
+                raise ValueError("File path must be set when using output_files.")
+
     def get_used_features(self) -> FeatureSet:
         return NOTHING
 

tests/test_collector.py

@@ -22,12 +22,14 @@
 from tested.serialisation import FunctionCall, FunctionType
 from tested.testsuite import (
     Context,
+    FileOutputChannel,
     MainInput,
     Output,
     Suite,
     SupportedLanguage,
     Tab,
     Testcase,
+    TextData,
 )
 from tests.manual_utils import assert_valid_output, configuration
 
@@ -196,3 +198,69 @@ def test_mid_tab_is_completed(tmp_path: Path, pytestconfig: pytest.Config):
         "wrong",
         "wrong",
     ]
+
+
+FILE_OUTPUT_SUITE = Suite(
+    tabs=[
+        Tab(
+            name="Tab 1",
+            contexts=[
+                Context(
+                    testcases=[
+                        Testcase(
+                            input=MainInput(arguments=["hello 1"]),
+                            output=Output(
+                                file=FileOutputChannel(
+                                    files=[
+                                        TextData(path="out1.txt", content="expected1"),
+                                        TextData(path="out2.txt", content="expected2"),
+                                    ]
+                                )
+                            ),
+                        ),
+                    ]
+                ),
+            ],
+        ),
+    ]
+)
+
+
+def test_complete_evaluation_with_file_output(
+    tmp_path: Path, pytestconfig: pytest.Config
+):
+    """Test that complete_evaluation emits per-file NOT_EXECUTED tests."""
+    conf = configuration(pytestconfig, "", SupportedLanguage.JAVASCRIPT, tmp_path)
+    result = StringIO()
+    bundle = create_bundle(conf, result, FILE_OUTPUT_SUITE)
+    collector = OutputManager(out=result)
+
+    # Open judgement but don't execute anything.
+    collector.add(StartJudgement())
+
+    terminate(bundle, collector, status_if_unclosed=Status.RUNTIME_ERROR)
+
+    updates = assert_valid_output(result.getvalue(), pytestconfig)
+
+    # Each file should produce its own NOT_EXECUTED test.
+    start_tests = updates.find_all("start-test")
+    assert len(start_tests) == 2
+    assert start_tests[0]["channel"] == "out1.txt"
+    assert start_tests[1]["channel"] == "out2.txt"
+
+    assert updates.find_status_enum() == [
+        "runtime error",
+        "wrong",
+        "wrong",
+    ]
+
+
+def test_file_output_channel_rejects_none_path():
+    """Test that FileOutputChannel rejects files with path=None."""
+    with pytest.raises(ValueError, match="File path must be set"):
+        FileOutputChannel(
+            files=[
+                TextData(path=None, content="expected1"),
+                TextData(path="out2.txt", content="expected2"),
+            ]
+        )