Skip to content

Fix importing a problem archive failed alert with "Array Array Array" #2639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 2, 2024
Merged

Fix importing a problem archive failed alert with "Array Array Array" #2639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
29f30ed
Clean up message handling after importing a zipped problem.
meisterT Nov 24, 2023
e99ea14
Skip parts of the integration tests that rely on v1.
meisterT May 11, 2024
9824c3a
Use ccs-specs repo for API check script and pin to a fixed commit.
eldering Jun 14, 2023
eadcb54
Use latest commit in CLICS 2023-06 version.
vmcj Jul 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions gitlab/ci_settings.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ export GITSHA
export PS4='(${BASH_SOURCE}:${LINENO}): - [$?] $ '
export LOGFILE="/opt/domjudge/domserver/webapp/var/log/prod.log"

CCS_SPECS_PINNED_SHA1='a68aff54c4e60fc2bff2fc5c36c119bffa4d30f1'

# Shared storage for all artifacts
export GITLABARTIFACTS="$DIR/gitlabartifacts"
mkdir -p "$GITLABARTIFACTS"
Expand Down
144 changes: 87 additions & 57 deletions gitlab/integration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,25 @@ mount
mount -o remount,exec,dev /builds
section_end mount

section_start check_cgroup_v1 "Checking for cgroup v1 availability"
grep cgroup$ /proc/filesystems
if [ $? -eq 0 ]; then
cgroupv1=1
else
echo "Skipping tests that rely on cgroup v1"
cgroupv1=0
fi
section_end check_cgroup_v1

section_start judgehost "Configure judgehost"
cd /opt/domjudge/judgehost/
sudo cp /opt/domjudge/judgehost/etc/sudoers-domjudge /etc/sudoers.d/
sudo chmod 400 /etc/sudoers.d/sudoers-domjudge
sudo bin/create_cgroups
if [ $cgroupv1 -ne 0 ]; then
# We allow this to go wrong as some gitlab runners do not have the
# swapaccount kernel option set.
sudo bin/create_cgroups || cgroupv1=0
fi

if [ ! -d ${DIR}/chroot/domjudge/ ]; then
cd ${DIR}/misc-tools
Expand All @@ -94,13 +108,15 @@ fi
section_end judgehost

section_start more_setup "Remaining setup (e.g. starting judgedaemon)"
# download domjudge-scripts for API check

# Download yajsv and ccs-specs for API check.
cd $HOME
composer -n require justinrainbow/json-schema
curl -o yajsv https://github.com/neilpa/yajsv/releases/download/v1.4.1/yajsv.linux.amd64
chmod a+x yajsv
echo -e "\033[0m"
PATH=${PATH}:${HOME}/vendor/bin
git clone --depth=1 https://github.com/DOMjudge/domjudge-scripts.git
CHECK_API=${HOME}/domjudge-scripts/contest-api/check-api.sh
git clone https://github.com/icpc/ccs-specs.git
( cd ccs-specs && git reset --hard $CCS_SPECS_PINNED_SHA1 )
CHECK_API="${HOME}/ccs-specs/check-api.sh -j ${HOME}/yajsv"

# Recreate domjudge-run-0 user with random UID to prevent clashes with
# existing users in the host and other CI jobs, which can lead to
Expand Down Expand Up @@ -128,8 +144,10 @@ set -e
if [ $PIN_JUDGEDAEMON -eq 1 ]; then
PINNING="-n 0"
fi
sudo -u domjudge bin/judgedaemon $PINNING |& tee /tmp/judgedaemon.log &
sleep 5
if [ $cgroupv1 -ne 0 ]; then
sudo -u domjudge bin/judgedaemon $PINNING |& tee /tmp/judgedaemon.log &
sleep 5
fi

section_end more_setup

Expand All @@ -154,9 +172,7 @@ for i in hello_kattis different guess; do
done
section_end submitting

section_start judging "Waiting until all submissions are judged"
# wait for and check results
NUMSUBS=$(curl --fail http://admin:$ADMINPASS@localhost/domjudge/api/contests/1/submissions | python3 -mjson.tool | grep -c '"id":')
section_start curlcookie "Preparing cookie jar for curl"
export COOKIEJAR
COOKIEJAR=$(mktemp --tmpdir)
export CURLOPTS="--fail -sq -m 30 -b $COOKIEJAR"
Expand All @@ -170,56 +186,64 @@ curl $CURLOPTS -c $COOKIEJAR -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F
curl $CURLOPTS -F "sendto=" -F "problem=1-" -F "bodytext=Testing" -F "submit=Send" \
"http://localhost/domjudge/jury/clarifications/send" -o /dev/null

# Don't spam the log.
set +x
section_end curlcookie

while /bin/true; do
sleep 30s
curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" -o /dev/null
if [ $cgroupv1 -ne 0 ]; then
section_start judging "Waiting until all submissions are judged"
# wait for and check results
NUMSUBS=$(curl --fail http://admin:$ADMINPASS@localhost/domjudge/api/contests/1/submissions | python3 -mjson.tool | grep -c '"id":')

# Check if we are done, i.e. everything is judged or something got disabled by internal error...
if tail /tmp/judgedaemon.log | grep -q "No submissions in queue"; then
break
fi
# ... or something has crashed.
if ! pgrep -f judgedaemon; then
break
fi
done
# Don't spam the log.
set +x

NUMNOTVERIFIED=$(curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions checked" | sed -r 's/^.* ([0-9]+) submissions checked.*$/\1/')
NUMVERIFIED=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions not checked" | sed -r 's/^.* ([0-9]+) submissions not checked.*$/\1/')
NUMNOMAGIC=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "without magic string" | sed -r 's/^.* ([0-9]+) without magic string.*$/\1/')
section_end judging

# We expect
# - two submissions with ambiguous outcome,
# - no submissions without magic string,
# - and all submissions to be judged.
if [ $NUMNOTVERIFIED -ne 2 ] || [ $NUMNOMAGIC -ne 0 ] || [ $NUMSUBS -gt $((NUMVERIFIED+NUMNOTVERIFIED)) ]; then
section_start error "Short error description"
# We error out below anyway, so no need to fail earlier than that.
set +e
echo "verified subs: $NUMVERIFIED, unverified subs: $NUMNOTVERIFIED, total subs: $NUMSUBS"
echo "(expected 2 submissions to be unverified, but all to be processed)"
echo "Of these $NUMNOMAGIC do not have the EXPECTED_RESULTS string (should be 0)."
curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" | w3m -dump -T text/html
section_end error

section_start logfiles "All the more or less useful logfiles"
for i in /opt/domjudge/judgehost/judgings/*/*/*/*/*/compile.out; do
echo $i;
head -n 100 $i;
dir=$(dirname $i)
if [ -r $dir/testcase001/system.out ]; then
head $dir/testcase001/system.out
head $dir/testcase001/runguard.err
head $dir/testcase001/program.err
head $dir/testcase001/program.meta
while /bin/true; do
sleep 30s
curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" -o /dev/null

# Check if we are done, i.e. everything is judged or something got disabled by internal error...
if tail /tmp/judgedaemon.log | grep -q "No submissions in queue"; then
break
fi
# ... or something has crashed.
if ! pgrep -f judgedaemon; then
break
fi
echo;
done
exit 1;

NUMNOTVERIFIED=$(curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions checked" | sed -r 's/^.* ([0-9]+) submissions checked.*$/\1/')
NUMVERIFIED=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions not checked" | sed -r 's/^.* ([0-9]+) submissions not checked.*$/\1/')
NUMNOMAGIC=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "without magic string" | sed -r 's/^.* ([0-9]+) without magic string.*$/\1/')
section_end judging

# We expect
# - two submissions with ambiguous outcome,
# - no submissions without magic string,
# - and all submissions to be judged.
if [ $NUMNOTVERIFIED -ne 2 ] || [ $NUMNOMAGIC -ne 0 ] || [ $NUMSUBS -gt $((NUMVERIFIED+NUMNOTVERIFIED)) ]; then
section_start error "Short error description"
# We error out below anyway, so no need to fail earlier than that.
set +e
echo "verified subs: $NUMVERIFIED, unverified subs: $NUMNOTVERIFIED, total subs: $NUMSUBS"
echo "(expected 2 submissions to be unverified, but all to be processed)"
echo "Of these $NUMNOMAGIC do not have the EXPECTED_RESULTS string (should be 0)."
curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" | w3m -dump -T text/html
section_end error

section_start logfiles "All the more or less useful logfiles"
for i in /opt/domjudge/judgehost/judgings/*/*/*/*/*/compile.out; do
echo $i;
head -n 100 $i;
dir=$(dirname $i)
if [ -r $dir/testcase001/system.out ]; then
head $dir/testcase001/system.out
head $dir/testcase001/runguard.err
head $dir/testcase001/program.err
head $dir/testcase001/program.meta
fi
echo;
done
exit 1;
fi
fi

section_start api_check "Performing API checks"
Expand All @@ -239,7 +263,13 @@ if cat /opt/domjudge/domserver/webapp/var/log/prod.log | egrep '(CRITICAL|ERROR)
fi

# Check the Contest API:
$CHECK_API -n -C -e -a 'strict=1' http://admin:$ADMINPASS@localhost/domjudge/api
if [ $cgroupv1 -ne 0 ]; then
$CHECK_API -n -C -e -a 'strict=1' http://admin:$ADMINPASS@localhost/domjudge/api
else
# With cgroup v1 not being available we don't judge, so we cannot do
# consistency checks, so running the above command without -C.
$CHECK_API -n -e -a 'strict=1' http://admin:$ADMINPASS@localhost/domjudge/api
fi
section_end api_check |& tee "$GITLABARTIFACTS/check_api.log"

section_start validate_feed "Validate the eventfeed against API (ignoring failures)"
Expand Down
21 changes: 14 additions & 7 deletions webapp/src/Controller/Jury/ImportExportController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ public function indexAction(Request $request): Response
$this->dj->auditlog('problem', $newProblem->getProbid(), 'upload zip',
$clientName);
} else {
$this->addFlash('danger', implode("\n", $allMessages));
$this->postMessages($allMessages);
return $this->redirectToRoute('jury_problems');
}
} catch (Exception $e) {
Expand All @@ -194,12 +194,7 @@ public function indexAction(Request $request): Response
$zip->close();
}
}

foreach (['info', 'warning', 'danger'] as $type) {
if (!empty($allMessages[$type])) {
$this->addFlash($type, implode("\n", $allMessages[$type]));
}
}
$this->postMessages($allMessages);

if ($newProblem !== null) {
return $this->redirectToRoute('jury_problem', ['probId' => $newProblem->getProbid()]);
Expand Down Expand Up @@ -575,4 +570,16 @@ protected function getClarificationsHtml(): Response
'problems' => $contestProblems,
]);
}

/**
* @param array<string, string[]> $allMessages
*/
private function postMessages(array $allMessages): void
{
foreach (['info', 'warning', 'danger'] as $type) {
if (!empty($allMessages[$type])) {
$this->addFlash($type, implode("\n", $allMessages[$type]));
}
}
}
}
19 changes: 13 additions & 6 deletions webapp/src/Controller/Jury/ProblemController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -972,12 +972,7 @@ public function editAction(Request $request, int $probId): Response
$zip->close();
}
}

foreach (['info', 'warning', 'danger'] as $type) {
if (!empty($messages[$type])) {
$this->addFlash($type, implode("\n", $messages[$type]));
}
}
$this->postMessages($messages);

return $this->redirectToRoute('jury_problem', ['probId' => $problem->getProbid()]);
}
Expand Down Expand Up @@ -1172,4 +1167,16 @@ public function requestRemainingRunsWholeProblemAction(string $probId): Redirect
$this->judgeRemaining($judgings);
return $this->redirect($this->generateUrl('jury_problem', ['probId' => $probId]));
}

/**
* @param array<string, string[]> $allMessages
*/
private function postMessages(array $allMessages): void
{
foreach (['info', 'warning', 'danger'] as $type) {
if (!empty($allMessages[$type])) {
$this->addFlash($type, implode("\n", $allMessages[$type]));
}
}
}
}
Loading