Skip to content

Commit 5c9d385

Browse files
meisterTmeisterT
committed
Basic implementation of partial scoring.
This follows the legacy spec in: https://icpc.io/problem-package-format/spec/legacy.html#graders We might decided at a later point whether we want to support this or just the new (currently draft) spec, but basic constructs of the change should be the same. Not yet implemented: - any form of testing - anything on the team interface - shadowing Part of #2518
1 parent b84190a commit 5c9d385

25 files changed

+861
-46
lines changed

‎judge/judgedaemon.main.php

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1461,6 +1461,7 @@ function judge(array $judgeTask): bool
14611461
}
14621462
}
14631463

1464+
$compare_args = $compare_config['compare_args'];
14641465
$test_run_cmd = LIBJUDGEDIR . "/testcase_run.sh $cpuset_opt " .
14651466
implode(' ', array_map('dj_escapeshellarg', [
14661467
$input,
@@ -1469,7 +1470,7 @@ function judge(array $judgeTask): bool
14691470
$passdir,
14701471
$run_runpath,
14711472
$compare_runpath,
1472-
$compare_config['compare_args']
1473+
$compare_args,
14731474
]));
14741475
system($test_run_cmd, $retval);
14751476

@@ -1523,11 +1524,16 @@ function judge(array $judgeTask): bool
15231524
if (file_exists($passdir . '/feedback/teammessage.txt')) {
15241525
$new_judging_run['team_message'] = rest_encode_file($passdir . '/feedback/teammessage.txt', $output_storage_limit);
15251526
}
1527+
$score = "";
1528+
if ($result === 'correct' && file_exists($passdir . '/feedback/score.txt')) {
1529+
$new_judging_run['score'] = rest_encode_file($passdir . '/feedback/score.txt');
1530+
$score = ", score: " . trim(dj_file_get_contents($passdir . '/feedback/score.txt'));
1531+
}
15261532

15271533
if ($passLimit > 1) {
15281534
$walltime = $metadata['wall-time'] ?? '?';
15291535
logmsg(LOG_INFO, ' ' . ($result === 'correct' ? " \033[0;32m✔\033[0m" : " \033[1;31m✗\033[0m")
1530-
. ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result);
1536+
. ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result . $score);
15311537
}
15321538

15331539
if ($result !== 'correct') {
@@ -1575,7 +1581,7 @@ function judge(array $judgeTask): bool
15751581
if ($passLimit == 1) {
15761582
$walltime = $metadata['wall-time'] ?? '?';
15771583
logmsg(LOG_INFO, ' ' . ($result === 'correct' ? " \033[0;32m✔\033[0m" : " \033[1;31m✗\033[0m")
1578-
. ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result);
1584+
. ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result . $score);
15791585
}
15801586

15811587
// done!

‎webapp/migrations/Version20250704181912.php

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace DoctrineMigrations;
6+
7+
use Doctrine\DBAL\Schema\Schema;
8+
use Doctrine\Migrations\AbstractMigration;
9+
10+
/**
11+
* Auto-generated Migration: Please modify to your needs!
12+
*/
13+
final class Version20250704181912 extends AbstractMigration
14+
{
15+
public function getDescription(): string
16+
{
17+
return 'Add basic support for partial scoring';
18+
}
19+
20+
public function up(Schema $schema): void
21+
{
22+
// this up() migration is auto-generated, please modify it to your needs
23+
$this->addSql('CREATE TABLE testcase_group (testcase_group_id INT UNSIGNED AUTO_INCREMENT NOT NULL COMMENT \'Testcase group ID\', name VARCHAR(255) NOT NULL COMMENT \'Name of the testcase group\', accept_score VARCHAR(255) DEFAULT NULL COMMENT \'Score if this group is accepted\', range_lower_bound VARCHAR(255) DEFAULT NULL COMMENT \'Lower bound of the score range\', range_upper_bound VARCHAR(255) DEFAULT NULL COMMENT \'Upper bound of the score range\', aggregation_type VARCHAR(255) DEFAULT \'sum\' NOT NULL COMMENT \'How to aggregate scores for this group\', ignore_sample TINYINT(1) DEFAULT 0 NOT NULL COMMENT \'Ignore the sample testcases when aggregating scores\', PRIMARY KEY(testcase_group_id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB COMMENT = \'Testcase group metadata\' ');
24+
$this->addSql('ALTER TABLE testcase ADD testcase_group_id INT UNSIGNED DEFAULT NULL COMMENT \'Testcase group ID\'');
25+
$this->addSql('ALTER TABLE testcase ADD CONSTRAINT FK_4C1E5C391FF421A3 FOREIGN KEY (testcase_group_id) REFERENCES testcase_group (testcase_group_id) ON DELETE SET NULL');
26+
$this->addSql('CREATE INDEX IDX_4C1E5C391FF421A3 ON testcase (testcase_group_id)');
27+
$this->addSql('ALTER TABLE judging_run ADD score NUMERIC(32, 9) DEFAULT \'0.000000000\' NOT NULL COMMENT \'Optional score for this run, e.g. for partial scoring\'');
28+
$this->addSql('ALTER TABLE judging ADD score NUMERIC(32, 9) DEFAULT \'0.000000000\' NOT NULL COMMENT \'Optional score for this run, e.g. for partial scoring\'');
29+
$this->addSql('ALTER TABLE testcase_group ADD parent_id INT UNSIGNED DEFAULT NULL COMMENT \'Testcase group ID\'');
30+
$this->addSql('ALTER TABLE testcase_group ADD CONSTRAINT FK_F02888FE727ACA70 FOREIGN KEY (parent_id) REFERENCES testcase_group (testcase_group_id) ON DELETE SET NULL');
31+
$this->addSql('CREATE INDEX IDX_F02888FE727ACA70 ON testcase_group (parent_id)');
32+
$this->addSql('ALTER TABLE problem ADD parent_testcase_group_id INT UNSIGNED DEFAULT NULL COMMENT \'Testcase group ID\'');
33+
$this->addSql('ALTER TABLE problem ADD CONSTRAINT FK_D7E7CCC8A090DCC7 FOREIGN KEY (parent_testcase_group_id) REFERENCES testcase_group (testcase_group_id) ON DELETE SET NULL');
34+
$this->addSql('CREATE INDEX IDX_D7E7CCC8A090DCC7 ON problem (parent_testcase_group_id)');
35+
$this->addSql('ALTER TABLE testcase_group ADD output_validator_flags VARCHAR(255) DEFAULT NULL COMMENT \'Flags for output validation\'');
36+
$this->addSql('ALTER TABLE testcase_group ADD on_reject_continue TINYINT(1) DEFAULT 0 NOT NULL COMMENT \'Continue on reject\'');
37+
$this->addSql('ALTER TABLE scorecache ADD score_public NUMERIC(32, 9) DEFAULT \'0.000000000\' NOT NULL COMMENT \'Optional score for this run, e.g. for partial scoring\', ADD score_restricted NUMERIC(32, 9) DEFAULT \'0.000000000\' NOT NULL COMMENT \'Optional score for this run, e.g. for partial scoring (for restricted audience)\'');
38+
$this->addSql('ALTER TABLE rankcache ADD score_public NUMERIC(32, 9) DEFAULT \'0.000000000\' NOT NULL COMMENT \'Optional score for this run, e.g. for partial scoring\', ADD score_restricted NUMERIC(32, 9) DEFAULT \'0.000000000\' NOT NULL COMMENT \'Optional score for this run, e.g. for partial scoring (for restricted audience)\'');
39+
}
40+
41+
public function down(Schema $schema): void
42+
{
43+
// this down() migration is auto-generated, please modify it to your needs
44+
$this->addSql('ALTER TABLE rankcache DROP score_public, DROP score_restricted');
45+
$this->addSql('ALTER TABLE scorecache DROP score_public, DROP score_restricted');
46+
$this->addSql('ALTER TABLE testcase_group DROP on_reject_continue');
47+
$this->addSql('ALTER TABLE testcase_group DROP output_validator_flags');
48+
$this->addSql('ALTER TABLE problem DROP FOREIGN KEY FK_D7E7CCC8A090DCC7');
49+
$this->addSql('DROP INDEX IDX_D7E7CCC8A090DCC7 ON problem');
50+
$this->addSql('ALTER TABLE problem DROP parent_testcase_group_id');
51+
$this->addSql('ALTER TABLE testcase_group DROP FOREIGN KEY FK_F02888FE727ACA70');
52+
$this->addSql('DROP INDEX IDX_F02888FE727ACA70 ON testcase_group');
53+
$this->addSql('ALTER TABLE testcase_group DROP parent_id');
54+
$this->addSql('ALTER TABLE judging DROP score');
55+
$this->addSql('ALTER TABLE judging_run DROP score');
56+
$this->addSql('ALTER TABLE testcase DROP FOREIGN KEY FK_4C1E5C391FF421A3');
57+
$this->addSql('DROP TABLE testcase_group');
58+
$this->addSql('DROP INDEX IDX_4C1E5C391FF421A3 ON testcase');
59+
$this->addSql('ALTER TABLE testcase DROP testcase_group_id');
60+
}
61+
62+
public function isTransactional(): bool
63+
{
64+
return false;
65+
}
66+
}

‎webapp/src/Controller/API/JudgehostController.php

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use App\Entity\Rejudging;
1919
use App\Entity\Submission;
2020
use App\Entity\SubmissionFile;
21+
use App\Entity\TestcaseAggregationType;
2122
use App\Entity\TestcaseContent;
2223
use App\Entity\Version;
2324
use App\Service\BalloonService;
@@ -652,15 +653,16 @@ public function addJudgingRunAction(
652653
$teamMessage = $request->request->get('team_message');
653654
$metadata = $request->request->get('metadata');
654655
$testcasedir = $request->request->get('testcasedir');
655-
$compareMeta = $request->request->get('compare_metadata');
656+
$compareMeta = $request->request->get('compare_metadata');
657+
$score = $request->request->get('score');
656658

657659
$judgehost = $this->em->getRepository(Judgehost::class)->findOneBy(['hostname' => $hostname]);
658660
if (!$judgehost) {
659661
throw new BadRequestHttpException("Who are you and why are you sending us any data?");
660662
}
661663

662664
$hasFinalResult = $this->addSingleJudgingRun($judgeTaskId, $hostname, $runResult, $runTime,
663-
$outputSystem, $outputError, $outputDiff, $outputRun, $teamMessage, $metadata, $testcasedir, $compareMeta);
665+
$outputSystem, $outputError, $outputDiff, $outputRun, $teamMessage, $metadata, $testcasedir, $compareMeta, $score);
664666
$judgehost = $this->em->getRepository(Judgehost::class)->findOneBy(['hostname' => $hostname]);
665667
$judgehost->setPolltime(Utils::now());
666668
$this->em->flush();
@@ -932,6 +934,7 @@ private function addSingleJudgingRun(
932934
string $metadata,
933935
?string $testcasedir,
934936
?string $compareMeta,
937+
?string $score = null
935938
): bool {
936939
$resultsRemap = $this->config->get('results_remap');
937940
$resultsPrio = $this->config->get('results_prio');
@@ -953,7 +956,8 @@ private function addSingleJudgingRun(
953956
$teamMessage,
954957
$metadata,
955958
$testcasedir,
956-
$compareMeta
959+
$compareMeta,
960+
$score
957961
) {
958962
$judgingRun = $this->em->getRepository(JudgingRun::class)->findOneBy(
959963
['judgetaskid' => $judgeTaskId]);
@@ -983,6 +987,10 @@ private function addSingleJudgingRun(
983987
$judgingRunOutput->setTeamMessage(base64_decode($teamMessage));
984988
}
985989

990+
if ($score) {
991+
$judgingRun->setScore(base64_decode($score));
992+
}
993+
986994
$judging = $judgingRun->getJudging();
987995
$this->maybeUpdateActiveJudging($judging);
988996
$this->em->flush();
@@ -1019,15 +1027,30 @@ private function addSingleJudgingRun(
10191027
$oldResult = $judging->getResult();
10201028

10211029
$lazyEval = DOMJudgeService::EVAL_LAZY;
1022-
if (($result = SubmissionService::getFinalResult($runresults, $resultsPrio)) !== null) {
1030+
$problem = $judging->getSubmission()->getProblem();
1031+
if ($problem->isScoringProblem()) {
1032+
$parentGroup = $problem->getParentTestcaseGroup();
1033+
$scoreAndResult = SubmissionService::maybeSetScoringResult(
1034+
$parentGroup,
1035+
$judging
1036+
);
1037+
$score = $scoreAndResult[0];
1038+
$result = $scoreAndResult[1];
1039+
} else {
1040+
$result = SubmissionService::getFinalResult($runresults, $resultsPrio);
1041+
}
1042+
if ($result !== null) {
10231043
// Lookup global lazy evaluation of results setting and possible problem specific override.
1024-
$lazyEval = $this->config->get('lazy_eval_results');
1044+
$lazyEval = $this->config->get('lazy_eval_results');
10251045
$problemLazy = $judging->getSubmission()->getContestProblem()->getLazyEvalResults();
10261046
if ($problemLazy !== DOMJudgeService::EVAL_DEFAULT) {
10271047
$lazyEval = $problemLazy;
10281048
}
10291049

10301050
$judging->setResult($result);
1051+
if ($problem->isScoringProblem()) {
1052+
$judging->setScore($score);
1053+
}
10311054

10321055
$hasNullResults = false;
10331056
foreach ($runresults as $runresult) {
@@ -1100,16 +1123,16 @@ private function addSingleJudgingRun(
11001123
}
11011124

11021125
$submission = $judging->getSubmission();
1103-
$contest = $submission->getContest();
1104-
$team = $submission->getTeam();
1105-
$problem = $submission->getProblem();
1126+
$contest = $submission->getContest();
1127+
$team = $submission->getTeam();
1128+
$problem = $submission->getProblem();
11061129
$this->scoreboardService->calculateScoreRow($contest, $team, $problem);
11071130

11081131
// We call alert here before possible validation. Note that
11091132
// this means that these alert messages should be treated as
11101133
// confidential information.
11111134
$msg = sprintf("submission %s, judging %s: %s",
1112-
$submission->getSubmitid(), $judging->getJudgingid(), $result);
1135+
$submission->getSubmitid(), $judging->getJudgingid(), $result);
11131136
$this->dj->alert($result === 'correct' ? 'accept' : 'reject', $msg);
11141137

11151138
// Potentially send a balloon, i.e. if no verification required (case of verification required is

‎webapp/src/Controller/BaseController.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
use App\Entity\ScoreCache;
1414
use App\Entity\Team;
1515
use App\Entity\TeamCategory;
16+
use App\Entity\TestcaseAggregationType;
1617
use App\Service\DOMJudgeService;
1718
use App\Service\EventLogService;
1819
use App\Utils\Utils;

‎webapp/src/Controller/Jury/ImportExportController.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ public function indexAction(Request $request): Response
157157
try {
158158
$zip = $this->dj->openZipFile($archive->getRealPath());
159159
$clientName = $archive->getClientOriginalName();
160+
/** @var array<string, string[]> $messages */
160161
$messages = [];
161162
if ($contestId === null) {
162163
$contest = null;

‎webapp/src/Controller/Jury/ProblemController.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -956,6 +956,7 @@ public function editAction(Request $request, int $probId): Response
956956
$data = $uploadForm->getData();
957957
/** @var UploadedFile $archive */
958958
$archive = $data['archive'];
959+
/** @var array<string, string[]> $messages */
959960
$messages = [];
960961

961962
/** @var Contest|null $contest */

‎webapp/src/Entity/Judging.php

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,28 @@ class Judging extends BaseApiEntity
176176
#[Serializer\Exclude]
177177
private ?InternalError $internalError = null;
178178

179+
#[ORM\Column(
180+
type: 'decimal',
181+
precision: 32,
182+
scale: 9,
183+
options: [
184+
'comment' => 'Optional score for this run, e.g. for partial scoring',
185+
'default' => '0.000000000',
186+
]
187+
)]
188+
private string|float $score = 0;
189+
190+
public function setScore(string|float $score): Judging
191+
{
192+
$this->score = $score;
193+
return $this;
194+
}
195+
196+
public function getScore(): string|float
197+
{
198+
return $this->score;
199+
}
200+
179201
public function getMaxRuntime(): ?float
180202
{
181203
if ($this->runs->isEmpty()) {

‎webapp/src/Entity/JudgingRun.php

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,17 @@ class JudgingRun extends BaseApiEntity
6060
#[Serializer\Exclude]
6161
private string|float|null $endtime = null;
6262

63+
#[ORM\Column(
64+
type: 'decimal',
65+
precision: 32,
66+
scale: 9,
67+
options: [
68+
'comment' => 'Optional score for this run, e.g. for partial scoring',
69+
'default' => '0.000000000',
70+
]
71+
)]
72+
private string|float $score = 0;
73+
6374
#[ORM\ManyToOne(inversedBy: 'runs')]
6475
#[ORM\JoinColumn(name: 'judgingid', referencedColumnName: 'judgingid', onDelete: 'CASCADE')]
6576
#[Serializer\Exclude]
@@ -162,6 +173,16 @@ public function getEndtime(): string|float|null
162173
return $this->endtime;
163174
}
164175

176+
public function setScore(string|float $score): JudgingRun
177+
{
178+
$this->score = $score;
179+
return $this;
180+
}
181+
public function getScore(): string|float
182+
{
183+
return $this->score;
184+
}
185+
165186
#[Serializer\VirtualProperty]
166187
#[Serializer\SerializedName('time')]
167188
#[Serializer\Type('string')]

‎webapp/src/Entity/Problem.php

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,11 @@ class Problem extends BaseApiEntity implements
211211
#[Serializer\Exclude]
212212
private Collection $languages;
213213

214+
#[ORM\ManyToOne(inversedBy: 'problems')]
215+
#[ORM\JoinColumn(name: 'parent_testcase_group_id', referencedColumnName: 'testcase_group_id', nullable: true, onDelete: 'SET NULL')]
216+
#[Serializer\Exclude]
217+
private ?TestcaseGroup $parentTestcaseGroup = null;
218+
214219
public function setProbid(int $probid): Problem
215220
{
216221
$this->probid = $probid;
@@ -654,4 +659,15 @@ public function removeLanguage(Language $language): Problem
654659
$this->languages->removeElement($language);
655660
return $this;
656661
}
662+
663+
public function setParentTestcaseGroup(?TestcaseGroup $parentTestcaseGroup): Problem
664+
{
665+
$this->parentTestcaseGroup = $parentTestcaseGroup;
666+
return $this;
667+
}
668+
669+
public function getParentTestcaseGroup(): ?TestcaseGroup
670+
{
671+
return $this->parentTestcaseGroup;
672+
}
657673
}

‎webapp/src/Entity/RankCache.php

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,28 @@ class RankCache
7979
)]
8080
private string $sortKeyRestricted = '';
8181

82+
#[ORM\Column(
83+
type: 'decimal',
84+
precision: 32,
85+
scale: 9,
86+
options: [
87+
'comment' => 'Optional score for this run, e.g. for partial scoring',
88+
'default' => '0.000000000',
89+
]
90+
)]
91+
private string|float $scorePublic = 0;
92+
93+
#[ORM\Column(
94+
type: 'decimal',
95+
precision: 32,
96+
scale: 9,
97+
options: [
98+
'comment' => 'Optional score for this run, e.g. for partial scoring (for restricted audience)',
99+
'default' => '0.000000000',
100+
]
101+
)]
102+
private string|float $scoreRestricted = 0;
103+
82104
public function setPointsRestricted(int $pointsRestricted): RankCache
83105
{
84106
$this->points_restricted = $pointsRestricted;
@@ -188,4 +210,26 @@ public function getSortKeyRestricted(): string
188210
{
189211
return $this->sortKeyRestricted;
190212
}
213+
214+
public function setScorePublic(string|float $scorePublic): RankCache
215+
{
216+
$this->scorePublic = $scorePublic;
217+
return $this;
218+
}
219+
220+
public function getScorePublic(): string|float
221+
{
222+
return $this->scorePublic;
223+
}
224+
225+
public function setScoreRestricted(string|float $scoreRestricted): RankCache
226+
{
227+
$this->scoreRestricted = $scoreRestricted;
228+
return $this;
229+
}
230+
231+
public function getScoreRestricted(): string|float
232+
{
233+
return $this->scoreRestricted;
234+
}
191235
}

0 commit comments

Comments
 (0)