Add a new batch-and-output task type.

Author: veluca93

cms/grading/steps/trusted.py

@@ -195,6 +195,7 @@ def checker_step(
     input_digest: str,
     correct_output_digest: str,
     output_filename: str,
+    extra_args: list[str] | None = None
 ) -> tuple[bool, float | None, list[str] | None]:
     """Run the explicit checker given by the admins
 
@@ -209,6 +210,7 @@ def checker_step(
         as "correct_output.txt".
     output_filename: inner filename of the user output (already in the
         sandbox).
+    extra_args: extra arguments to pass to the checker.
 
     return: success (true if the checker was able to check the solution
         successfully), outcome and text (both None if success is False).
@@ -240,7 +242,7 @@ def checker_step(
     command = ["./%s" % CHECKER_FILENAME,
                CHECKER_INPUT_FILENAME,
                CHECKER_CORRECT_OUTPUT_FILENAME,
-               output_filename]
+               output_filename] + (extra_args if extra_args is not None else [])
     box_success, success, unused_stats = trusted_step(sandbox, [command])
     if not box_success or not success:
         logger.error("Sandbox failed during checker step. "

cms/grading/tasktypes/Batch.py

@@ -69,6 +69,8 @@ class Batch(TaskType):
     input, correct output and user output) and should write the
     outcome to stdout and the text to stderr.
 
+    Note that this class is used as a base class for the BatchAndOutput task
+    type.
     """
     # Codename of the checker, if it is used.
     CHECKER_CODENAME = "checker"
@@ -114,7 +116,7 @@ class Batch(TaskType):
     ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION]
 
     @property
-    def name(self):
+    def name(self) -> str:
         """See TaskType.name."""
         # TODO add some details if a grader/comparator is used, etc...
         return "Batch"
@@ -145,7 +147,10 @@ def get_compilation_commands(self, submission_format):
         codenames_to_compile = []
         if self._uses_grader():
             codenames_to_compile.append(self.GRADER_BASENAME + ".%l")
-        codenames_to_compile.extend(submission_format)
+        # For regular batch, all parts of the submission format end with %l.
+        # For batch+output only, some might not.
+        codenames_to_compile.extend(
+            [x for x in submission_format if x.endswith('.%l')])
         res = dict()
         for language in LANGUAGES:
             source_ext = language.source_extension
@@ -187,18 +192,14 @@ def _executable_filename(codenames: Iterable[str], language: Language) -> str:
         return: a deterministic executable name.
 
         """
-        name =  "_".join(sorted(codename.replace(".%l", "")
-                                for codename in codenames))
+        name = "_".join(sorted(codename.replace(".%l", "")
+                               for codename in codenames))
         return name + language.executable_extension
 
-    def compile(self, job, file_cacher):
-        """See TaskType.compile."""
+    def _do_compile(self, job, file_cacher):
         language = get_language(job.language)
         source_ext = language.source_extension
 
-        if not check_files_number(job, 1, or_more=True):
-            return
-
         # Create the list of filenames to be passed to the compiler. If we use
         # a grader, it needs to be in first position in the command line, and
         # we check that it exists.
@@ -215,6 +216,8 @@ def compile(self, job, file_cacher):
                 job.managers[grader_filename].digest
         # User's submitted file(s) (copy and add to compilation).
         for codename, file_ in job.files.items():
+            if not codename.endswith(".%l"):
+                continue
             filename = codename.replace(".%l", source_ext)
             filenames_to_compile.append(filename)
             filenames_and_digests_to_get[filename] = file_.digest
@@ -256,16 +259,19 @@ def compile(self, job, file_cacher):
         # Cleanup.
         delete_sandbox(sandbox, job.success, job.keep_sandbox)
 
-    def evaluate(self, job, file_cacher):
-        """See TaskType.evaluate."""
-        if not check_executables_number(job, 1):
+    def compile(self, job, file_cacher):
+        """See TaskType.compile."""
+        if not check_files_number(job, 1, or_more=True):
             return
 
+        self._do_compile(job, file_cacher)
+
+    def _execution_step(self, job, file_cacher):
         # Prepare the execution
         executable_filename = next(iter(job.executables.keys()))
         language = get_language(job.language)
         main = self.GRADER_BASENAME if self._uses_grader() \
-               else os.path.splitext(executable_filename)[0]
+            else os.path.splitext(executable_filename)[0]
         commands = language.get_evaluation_commands(
             executable_filename, main=main)
         executables_to_get = {
@@ -311,6 +317,7 @@ def evaluate(self, job, file_cacher):
 
         outcome = None
         text = None
+        output_file_params = None
 
         # Error in the sandbox: nothing to do!
         if not box_success:
@@ -347,20 +354,41 @@ def evaluate(self, job, file_cacher):
                     outcome = 0.0
                     text = [N_("Execution completed successfully")]
 
-                # Otherwise evaluate the output file.
+                # Otherwise prepare to evaluate the output file.
                 else:
-                    box_success, outcome, text = eval_output(
-                        file_cacher, job,
-                        self.CHECKER_CODENAME
-                        if self._uses_checker() else None,
-                        user_output_path=sandbox.relative_path(
+                    output_file_params = {
+                        'user_output_path': sandbox.relative_path(
                             self._actual_output),
-                        user_output_filename=self.output_filename)
+                        'user_output_filename': self.output_filename}
+
+        return outcome, text, output_file_params, stats, box_success, sandbox
+
+    def _evaluate_step(self, job, file_cacher, output_file_params, outcome, text, stats, box_success, sandbox, extra_args):
+        if box_success:
+            assert (output_file_params is None) == (outcome is not None)
+            if output_file_params is not None:
+                box_success, outcome, text = eval_output(
+                    file_cacher, job,
+                    self.CHECKER_CODENAME
+                    if self._uses_checker() else None,
+                    **output_file_params, extra_args=extra_args)
 
         # Fill in the job with the results.
         job.success = box_success
         job.outcome = str(outcome) if outcome is not None else None
         job.text = text
         job.plus = stats
 
-        delete_sandbox(sandbox, job.success, job.keep_sandbox)
+        if sandbox is not None:
+            delete_sandbox(sandbox, job.success, job.keep_sandbox)
+
+    def evaluate(self, job, file_cacher):
+        """See TaskType.evaluate."""
+        if not check_executables_number(job, 1):
+            return
+
+        outcome, text, output_file_params, stats, box_success, sandbox = self._execution_step(
+            job, file_cacher)
+
+        self._evaluate_step(job, file_cacher, output_file_params,
+                            outcome, text, stats, box_success,  sandbox, extra_args=None)

cms/grading/tasktypes/BatchAndOutput.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+
+# Contest Management System - http://cms-dev.github.io/
+# Copyright © 2010-2015 Giovanni Mascellani <[email protected]>
+# Copyright © 2010-2018 Stefano Maggiolo <[email protected]>
+# Copyright © 2010-2012 Matteo Boscariol <[email protected]>
+# Copyright © 2012-2014 Luca Wehrstedt <[email protected]>
+# Copyright © 2017 Myungwoo Chun <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+from cms.grading.ParameterTypes import ParameterTypeString
+from .Batch import Batch
+
+
+logger = logging.getLogger(__name__)
+
+
+# Dummy function to mark translatable string.
+def N_(message):
+    return message
+
+
+class BatchAndOutput(Batch):
+    """Task type class for a task that is a combination of Batch and
+    OutputOnly.
+
+    Parameters needs to be a list of four elements.
+
+    The first element is 'grader' or 'alone': in the first
+    case, the source file is to be compiled with a provided piece of
+    software ('grader'); in the other by itself.
+
+    The second element is a 2-tuple of the input file name and output file
+    name. The input file may be '' to denote stdin, and similarly the
+    output filename may be '' to denote stdout.
+
+    The third element is 'diff' or 'comparator' and says whether the
+    output is compared with a simple diff algorithm or using a
+    comparator.
+
+    The fourth element specifies testcases that *must* be provided as
+    output-only. If a testcase is not in this list, and is not provided
+    explicitly, then the provided source file will be used to compute
+    the output.
+
+    Note: the first element is used only in the compilation step; the
+    others only in the evaluation step.
+
+    A comparator can read argv[1], argv[2], argv[3], and argv[4] (respectively,
+    input, correct output, user output and 'outputonly' if the testcase was an
+    outputonly-only testcase, 'batch' otherwise) and should write the outcome
+    to stdout and the text to stderr.
+
+    The submission format for tasks of this task type should contain both
+    a single source file and one or more text files named "output_%s.txt",
+    where "%s" is the test case index.
+
+    """
+
+    # Template for the filename of the output files provided by the user; %s
+    # represent the testcase codename.
+    USER_OUTPUT_FILENAME_TEMPLATE = "output_%s.txt"
+
+    # Other constants to specify the task type behaviour and parameters.
+    ALLOW_PARTIAL_SUBMISSION = True
+
+    _OUTPUT_ONLY_TESTCASES = ParameterTypeString(
+        "Comma-separated list of output only testcases",
+        "output_only_testcases",
+        "")
+
+    ACCEPTED_PARAMETERS = Batch.ACCEPTED_PARAMETERS + [_OUTPUT_ONLY_TESTCASES]
+
+    @property
+    def name(self):
+        """See TaskType.name."""
+        # TODO add some details if a grader/comparator is used, etc...
+        return "BatchAndOutput"
+
+    def __init__(self, parameters):
+        super().__init__(parameters)
+
+        # Data in the parameters that is not in Batch.
+        self.output_only_testcases: set[str] = set(
+            self.parameters[3].split(','))
+
+    @staticmethod
+    def _get_user_output_filename(job):
+        return BatchAndOutput.USER_OUTPUT_FILENAME_TEMPLATE % \
+            job.operation.testcase_codename
+
+    def compile(self, job, file_cacher):
+        """See TaskType.compile."""
+        num_source_files = 0
+        for (codename, _) in job.files.items():
+            if codename.endswith(".%l"):
+                num_source_files += 1
+
+        if num_source_files == 0:
+            # This submission did not have any source files, skip compilation
+            job.success = True
+            job.compilation_success = True
+            job.text = [N_("No compilation needed")]
+            job.plus = {}
+            return
+
+        self._do_compile(job, file_cacher)
+
+    def evaluate(self, job, file_cacher):
+        """See TaskType.evaluate."""
+
+        user_output_filename = self._get_user_output_filename(job)
+
+        output_file_params = None
+        sandbox = None
+        outcome = None
+        text = ""
+        stats = {}
+        box_success = True
+
+        if user_output_filename in job.files:
+            output_file_params = {
+                'user_output_digest': job.files[user_output_filename].digest}
+        elif job.operation.testcase_codename in self.output_only_testcases:
+            pass
+        elif job.executables:
+            outcome, text, output_file_params, stats, box_success, sandbox = self._execution_step(
+                job, file_cacher)
+
+        if output_file_params is None and outcome is None:
+            job.success = True
+            job.outcome = "0.0"
+            job.text = [N_("File not submitted")]
+            job.plus = {}
+            return
+
+        if job.operation.testcase_codename in self.output_only_testcases:
+            extra_args = ['outputonly']
+        else:
+            extra_args = ['batch']
+
+        self._evaluate_step(job, file_cacher, output_file_params,
+                            outcome, text, stats, box_success, sandbox, extra_args)

cms/grading/tasktypes/util.py

@@ -211,6 +211,7 @@ def eval_output(
     user_output_path: str | None = None,
     user_output_digest: str | None = None,
     user_output_filename: str = "",
+    extra_args: list[str] | None = None
 ) -> tuple[bool, float | None, list[str] | None]:
     """Evaluate ("check") a user output using a white diff or a checker.
 
@@ -224,6 +225,7 @@ def eval_output(
         using the path (exactly one must be non-None).
     user_output_filename: the filename the user was expected to write to,
         or empty if stdout (used to return an error to the user).
+    extra_args: additional arguments to pass to the checker
 
     return: tuple of success (true if the checker was
         able to check the solution successfully), outcome and text (both None
@@ -266,7 +268,7 @@ def eval_output(
             if checker_codename in job.managers else None
         success, outcome, text = checker_step(
             sandbox, checker_digest, job.input, job.output,
-            EVAL_USER_OUTPUT_FILENAME)
+            EVAL_USER_OUTPUT_FILENAME, extra_args)
 
         delete_sandbox(sandbox, success, job.keep_sandbox)
         return success, outcome, text