Implement analyst evaluation mode

The idea behind this mode is that we are not really shadowing and trust
the event feed results. However, we want to judge "interesting" runs
locally to get useful information without the judging capacity to judge
all testcases due to limited judgehost assignment.

We do not consider 'TLE' or 'AC' interesting, as rerunning will not
yield much more information. We consider 'WA' very interesting and
prioritize the judging, but allow manual judging to overtake the
priority. We consider 'CE' somewhat interesting, but downprioritize them
a lot. For other verdicts, keep the normal priority.

Author: Kevinjil

etc/db-config.yaml

@@ -118,12 +118,13 @@
             type: int
             default_value: 1
             public: false
-            description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging and is typically used when running as analyst system.
+            description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging. Analyst mode tries to judge only interesting testcases.
             options:
                 1: Lazy
                 2: Full judging
                 3: Only on request
-            regex: /^[123]$/
+                4: Analyst mode
+            regex: /^[1234]$/
             error_message: A value between 1 and 3 is required.
         -   name: judgehost_warning
             type: int

webapp/src/Controller/API/JudgehostController.php

@@ -1083,7 +1083,9 @@ private function addSingleJudgingRun(
                     throw new BadMethodCallException('internal bug: the evaluated result changed during judging');
                 }
 
-                if ($lazyEval !== DOMJudgeService::EVAL_FULL) {
+                if ($$lazyEval === DOMJudgeService::EVAL_ANALYST) {
+                    // Explicitly do not update priorities or cancel activated tasks.
+                } elseif ($lazyEval !== DOMJudgeService::EVAL_FULL) {
                     // We don't want to continue on this problem, even if there's spare resources.
                     $this->em->getConnection()->executeStatement(
                         'UPDATE judgetask SET valid=0, priority=:priority'

webapp/src/Service/DOMJudgeService.php

@@ -74,6 +74,7 @@ class DOMJudgeService
     final public const EVAL_LAZY = 1;
     final public const EVAL_FULL = 2;
     final public const EVAL_DEMAND = 3;
+    final public const EVAL_ANALYST = 4;
 
     // Regex external identifiers must adhere to. Note that we are not checking whether it
     // does not start with a dot or dash or ends with a dot. We could but it would make the
@@ -1184,7 +1185,7 @@ public function unblockJudgeTasks(): void
         }
     }
 
-    public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0): void
+    public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0, bool $valid = true): void
     {
         $submission = $judging->getSubmission();
         $problem    = $submission->getContestProblem();
@@ -1197,7 +1198,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas
             return;
         }
 
-        $this->actuallyCreateJudgetasks($priority, $judging, $overshoot);
+        $this->actuallyCreateJudgetasks($priority, $judging, $overshoot, $valid);
 
         $team = $submission->getTeam();
         $result = $this->em->createQueryBuilder()
@@ -1215,7 +1216,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas
 
         // Teams that submit frequently slow down the judge queue but should not be able to starve other teams of their
         // deserved and timely judgement.
-        // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definiition of "recent"
+        // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definition of "recent"
         // includes all submissions that have been placed at a virtual time (including penalty) more recent than 60s
         // ago. This is done in order to avoid punishing teams who submit while their submissions are stuck in the queue
         // for other reasons, for example an internal error for a problem or language.
@@ -1586,19 +1587,20 @@ private function allowJudge(ContestProblem $problem, Submission $submission, Lan
         return !$evalOnDemand;
     }
 
-    private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0): void
+    private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0, bool $valid = true): void
     {
         $submission = $judging->getSubmission();
         $problem    = $submission->getContestProblem();
         // We use a mass insert query, since that is way faster than doing a separate insert for each testcase.
-        // We first insert judgetasks, then select their ID's and finally insert the judging runs.
+        // We first insert judgetasks, then select their IDs and finally insert the judging runs.
 
         // Step 1: Create the template for the judgetasks.
         $compileExecutable = $submission->getLanguage()->getCompileExecutable()->getImmutableExecutable();
         $judgetaskInsertParams = [
             ':type' => JudgeTaskType::JUDGING_RUN,
             ':submitid' => $submission->getSubmitid(),
             ':priority' => $priority,
+            ':valid' => $valid ? 1 : 0,
             ':jobid' => $judging->getJudgingid(),
             ':uuid' => $judging->getUuid(),
             ':compile_script_id' => $compileExecutable->getImmutableExecId(),

webapp/src/Service/ExternalContestSourceService.php

@@ -27,6 +27,7 @@
 use App\Entity\ExternalJudgement;
 use App\Entity\ExternalRun;
 use App\Entity\ExternalSourceWarning;
+use App\Entity\JudgeTask;
 use App\Entity\Language;
 use App\Entity\Problem;
 use App\Entity\Submission;
@@ -1777,13 +1778,13 @@ protected function importRun(Event $event, EventData $data): void
         }
 
         // First, load the external run.
+        $persist = false;
         $run     = $this->em
             ->getRepository(ExternalRun::class)
             ->findOneBy([
                             'contest'    => $this->getSourceContest(),
                             'externalid' => $runId,
                         ]);
-        $persist = false;
         if (!$run) {
             $run = new ExternalRun();
             $run
@@ -1858,9 +1859,50 @@ protected function importRun(Event $event, EventData $data): void
         if ($persist) {
             $this->em->persist($run);
         }
+
+        $lazyEval = $this->config->get('lazy_eval_results');
+        if ($lazyEval === DOMJudgeService::EVAL_ANALYST) {
+            // Check if we want to judge this testcase locally to provide useful information for analysts
+            $priority = $this->getAnalystRunPriority($run);
+            if ($priority !== null) {
+                // Make the judgetask valid and assign running priority if no judgehost has picked it up yet.
+                $this->em->createQueryBuilder()
+                        ->update(JudgeTask::class, 'jt')
+                        ->set('jt.valid', true)
+                        ->set('jt.priority', $priority)
+                        ->andWhere('jt.testcase_id = :testcase_id')
+                        ->andWhere('jt.submission = :submission')
+                        ->andWhere('jt.judgehost IS NULL')
+                        ->setParameter('testcase_id', $testcase->getTestcaseid())
+                        ->setParameter('submission', $externalJudgement->getSubmission())
+                        ->getQuery()
+                        ->execute();
+            }
+        }
+
         $this->em->flush();
     }
 
+    /**
+     * Checks if this run is interesting to judge locally for more analysis results.
+     * @param ExternalRun $run
+     * @return int The judging priority if it should be run locally, null otherwise.
+     */
+    protected function getAnalystRunPriority(ExternalRun $run): int | null {
+        return match ($run->getResult()) {
+            // We will not get any new useful information for TLE testcases, while they take a lot of judgedaemon time.
+            'timelimit' => null,
+            // We often do not get new useful information for judging correct testcases.
+            'correct' => null,
+            // Wrong answers are interesting for the analysts, assign a high priority but below manual judging.
+            'wrong-answer' => -9,
+            // Compile errors could be interesting to see what went wrong, assign a low priority.
+            'compiler-error' => 9,
+            // Otherwise, judge with normal priority.
+            default => 0,
+        };
+    }
+
     protected function processPendingEvents(string $type, string|int $id): void
     {
         // Process pending events.

webapp/src/Service/SubmissionService.php

@@ -732,8 +732,15 @@ public function submitSolution(
             // This is so that we can use the submitid/judgingid below.
             $this->em->flush();
 
-            $this->dj->maybeCreateJudgeTasks($judging,
-                $source === SubmissionSource::PROBLEM_IMPORT ? JudgeTask::PRIORITY_LOW : JudgeTask::PRIORITY_DEFAULT);
+            $priority = match ($source) {
+                SubmissionSource::PROBLEM_IMPORT => JudgeTask::PRIORITY_LOW,
+                default => JudgeTask::PRIORITY_DEFAULT,
+            };
+            // Create judgetask as invalid when evaluating as analyst.
+            $lazyEval = $this->config->get('lazy_eval_results');
+            // We create invalid judgetasks, and only mark them valid when they are interesting for the analysts.
+            $start_invalid = $lazyEval === DOMJudgeService::EVAL_ANALYST && $source == SubmissionSource::SHADOWING;
+            $this->dj->maybeCreateJudgeTasks($judging, $priority, valid: !$start_invalid);
         }
 
         $this->em->wrapInTransaction(function () use ($contest, $submission) {

webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php

@@ -60,6 +60,7 @@ protected function setUp(): void
             'shadow_mode' => 0,
             'sourcefiles_limit' => 1,
             'sourcesize_limit' => 1024*256,
+            'lazy_eval_results' => 1,
         ];
 
         $this->config = $this->createMock(ConfigurationService::class);