diff --git a/gitlab/ci_settings.sh b/gitlab/ci_settings.sh index ad1eef947a..dfea8a1925 100755 --- a/gitlab/ci_settings.sh +++ b/gitlab/ci_settings.sh @@ -13,6 +13,8 @@ export GITSHA export PS4='(${BASH_SOURCE}:${LINENO}): - [$?] $ ' export LOGFILE="/opt/domjudge/domserver/webapp/var/log/prod.log" +CCS_SPECS_PINNED_SHA1='a68aff54c4e60fc2bff2fc5c36c119bffa4d30f1' + # Shared storage for all artifacts export GITLABARTIFACTS="$DIR/gitlabartifacts" mkdir -p "$GITLABARTIFACTS" diff --git a/gitlab/integration.sh b/gitlab/integration.sh index 1b470f0e50..5a75f90819 100755 --- a/gitlab/integration.sh +++ b/gitlab/integration.sh @@ -81,11 +81,25 @@ mount mount -o remount,exec,dev /builds section_end mount +section_start check_cgroup_v1 "Checking for cgroup v1 availability" +grep cgroup$ /proc/filesystems +if [ $? -eq 0 ]; then + cgroupv1=1 +else + echo "Skipping tests that rely on cgroup v1" + cgroupv1=0 +fi +section_end check_cgroup_v1 + section_start judgehost "Configure judgehost" cd /opt/domjudge/judgehost/ sudo cp /opt/domjudge/judgehost/etc/sudoers-domjudge /etc/sudoers.d/ sudo chmod 400 /etc/sudoers.d/sudoers-domjudge -sudo bin/create_cgroups +if [ $cgroupv1 -ne 0 ]; then + # We allow this to go wrong as some gitlab runners do not have the + # swapaccount kernel option set. + sudo bin/create_cgroups || cgroupv1=0 +fi if [ ! -d ${DIR}/chroot/domjudge/ ]; then cd ${DIR}/misc-tools @@ -94,13 +108,15 @@ fi section_end judgehost section_start more_setup "Remaining setup (e.g. starting judgedaemon)" -# download domjudge-scripts for API check + +# Download yajsv and ccs-specs for API check. cd $HOME -composer -n require justinrainbow/json-schema +curl -o yajsv https://github.com/neilpa/yajsv/releases/download/v1.4.1/yajsv.linux.amd64 +chmod a+x yajsv echo -e "\033[0m" -PATH=${PATH}:${HOME}/vendor/bin -git clone --depth=1 https://github.com/DOMjudge/domjudge-scripts.git -CHECK_API=${HOME}/domjudge-scripts/contest-api/check-api.sh +git clone https://github.com/icpc/ccs-specs.git +( cd ccs-specs && git reset --hard $CCS_SPECS_PINNED_SHA1 ) +CHECK_API="${HOME}/ccs-specs/check-api.sh -j ${HOME}/yajsv" # Recreate domjudge-run-0 user with random UID to prevent clashes with # existing users in the host and other CI jobs, which can lead to @@ -128,8 +144,10 @@ set -e if [ $PIN_JUDGEDAEMON -eq 1 ]; then PINNING="-n 0" fi -sudo -u domjudge bin/judgedaemon $PINNING |& tee /tmp/judgedaemon.log & -sleep 5 +if [ $cgroupv1 -ne 0 ]; then + sudo -u domjudge bin/judgedaemon $PINNING |& tee /tmp/judgedaemon.log & + sleep 5 +fi section_end more_setup @@ -154,9 +172,7 @@ for i in hello_kattis different guess; do done section_end submitting -section_start judging "Waiting until all submissions are judged" -# wait for and check results -NUMSUBS=$(curl --fail http://admin:$ADMINPASS@localhost/domjudge/api/contests/1/submissions | python3 -mjson.tool | grep -c '"id":') +section_start curlcookie "Preparing cookie jar for curl" export COOKIEJAR COOKIEJAR=$(mktemp --tmpdir) export CURLOPTS="--fail -sq -m 30 -b $COOKIEJAR" @@ -170,56 +186,64 @@ curl $CURLOPTS -c $COOKIEJAR -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F curl $CURLOPTS -F "sendto=" -F "problem=1-" -F "bodytext=Testing" -F "submit=Send" \ "http://localhost/domjudge/jury/clarifications/send" -o /dev/null -# Don't spam the log. -set +x +section_end curlcookie -while /bin/true; do - sleep 30s - curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" -o /dev/null +if [ $cgroupv1 -ne 0 ]; then + section_start judging "Waiting until all submissions are judged" + # wait for and check results + NUMSUBS=$(curl --fail http://admin:$ADMINPASS@localhost/domjudge/api/contests/1/submissions | python3 -mjson.tool | grep -c '"id":') - # Check if we are done, i.e. everything is judged or something got disabled by internal error... - if tail /tmp/judgedaemon.log | grep -q "No submissions in queue"; then - break - fi - # ... or something has crashed. - if ! pgrep -f judgedaemon; then - break - fi -done + # Don't spam the log. + set +x -NUMNOTVERIFIED=$(curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions checked" | sed -r 's/^.* ([0-9]+) submissions checked.*$/\1/') -NUMVERIFIED=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions not checked" | sed -r 's/^.* ([0-9]+) submissions not checked.*$/\1/') -NUMNOMAGIC=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "without magic string" | sed -r 's/^.* ([0-9]+) without magic string.*$/\1/') -section_end judging - -# We expect -# - two submissions with ambiguous outcome, -# - no submissions without magic string, -# - and all submissions to be judged. -if [ $NUMNOTVERIFIED -ne 2 ] || [ $NUMNOMAGIC -ne 0 ] || [ $NUMSUBS -gt $((NUMVERIFIED+NUMNOTVERIFIED)) ]; then - section_start error "Short error description" - # We error out below anyway, so no need to fail earlier than that. - set +e - echo "verified subs: $NUMVERIFIED, unverified subs: $NUMNOTVERIFIED, total subs: $NUMSUBS" - echo "(expected 2 submissions to be unverified, but all to be processed)" - echo "Of these $NUMNOMAGIC do not have the EXPECTED_RESULTS string (should be 0)." - curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" | w3m -dump -T text/html - section_end error - - section_start logfiles "All the more or less useful logfiles" - for i in /opt/domjudge/judgehost/judgings/*/*/*/*/*/compile.out; do - echo $i; - head -n 100 $i; - dir=$(dirname $i) - if [ -r $dir/testcase001/system.out ]; then - head $dir/testcase001/system.out - head $dir/testcase001/runguard.err - head $dir/testcase001/program.err - head $dir/testcase001/program.meta + while /bin/true; do + sleep 30s + curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" -o /dev/null + + # Check if we are done, i.e. everything is judged or something got disabled by internal error... + if tail /tmp/judgedaemon.log | grep -q "No submissions in queue"; then + break + fi + # ... or something has crashed. + if ! pgrep -f judgedaemon; then + break fi - echo; done - exit 1; + + NUMNOTVERIFIED=$(curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions checked" | sed -r 's/^.* ([0-9]+) submissions checked.*$/\1/') + NUMVERIFIED=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "submissions not checked" | sed -r 's/^.* ([0-9]+) submissions not checked.*$/\1/') + NUMNOMAGIC=$( curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier" | grep "without magic string" | sed -r 's/^.* ([0-9]+) without magic string.*$/\1/') + section_end judging + + # We expect + # - two submissions with ambiguous outcome, + # - no submissions without magic string, + # - and all submissions to be judged. + if [ $NUMNOTVERIFIED -ne 2 ] || [ $NUMNOMAGIC -ne 0 ] || [ $NUMSUBS -gt $((NUMVERIFIED+NUMNOTVERIFIED)) ]; then + section_start error "Short error description" + # We error out below anyway, so no need to fail earlier than that. + set +e + echo "verified subs: $NUMVERIFIED, unverified subs: $NUMNOTVERIFIED, total subs: $NUMSUBS" + echo "(expected 2 submissions to be unverified, but all to be processed)" + echo "Of these $NUMNOMAGIC do not have the EXPECTED_RESULTS string (should be 0)." + curl $CURLOPTS "http://localhost/domjudge/jury/judging-verifier?verify_multiple=1" | w3m -dump -T text/html + section_end error + + section_start logfiles "All the more or less useful logfiles" + for i in /opt/domjudge/judgehost/judgings/*/*/*/*/*/compile.out; do + echo $i; + head -n 100 $i; + dir=$(dirname $i) + if [ -r $dir/testcase001/system.out ]; then + head $dir/testcase001/system.out + head $dir/testcase001/runguard.err + head $dir/testcase001/program.err + head $dir/testcase001/program.meta + fi + echo; + done + exit 1; + fi fi section_start api_check "Performing API checks" @@ -239,7 +263,13 @@ if cat /opt/domjudge/domserver/webapp/var/log/prod.log | egrep '(CRITICAL|ERROR) fi # Check the Contest API: -$CHECK_API -n -C -e -a 'strict=1' http://admin:$ADMINPASS@localhost/domjudge/api +if [ $cgroupv1 -ne 0 ]; then + $CHECK_API -n -C -e -a 'strict=1' http://admin:$ADMINPASS@localhost/domjudge/api +else + # With cgroup v1 not being available we don't judge, so we cannot do + # consistency checks, so running the above command without -C. + $CHECK_API -n -e -a 'strict=1' http://admin:$ADMINPASS@localhost/domjudge/api +fi section_end api_check |& tee "$GITLABARTIFACTS/check_api.log" section_start validate_feed "Validate the eventfeed against API (ignoring failures)" diff --git a/webapp/src/Controller/Jury/ImportExportController.php b/webapp/src/Controller/Jury/ImportExportController.php index f1fa4f9b8f..2092b1d8c8 100644 --- a/webapp/src/Controller/Jury/ImportExportController.php +++ b/webapp/src/Controller/Jury/ImportExportController.php @@ -184,7 +184,7 @@ public function indexAction(Request $request): Response $this->dj->auditlog('problem', $newProblem->getProbid(), 'upload zip', $clientName); } else { - $this->addFlash('danger', implode("\n", $allMessages)); + $this->postMessages($allMessages); return $this->redirectToRoute('jury_problems'); } } catch (Exception $e) { @@ -194,12 +194,7 @@ public function indexAction(Request $request): Response $zip->close(); } } - - foreach (['info', 'warning', 'danger'] as $type) { - if (!empty($allMessages[$type])) { - $this->addFlash($type, implode("\n", $allMessages[$type])); - } - } + $this->postMessages($allMessages); if ($newProblem !== null) { return $this->redirectToRoute('jury_problem', ['probId' => $newProblem->getProbid()]); @@ -575,4 +570,16 @@ protected function getClarificationsHtml(): Response 'problems' => $contestProblems, ]); } + + /** + * @param array $allMessages + */ + private function postMessages(array $allMessages): void + { + foreach (['info', 'warning', 'danger'] as $type) { + if (!empty($allMessages[$type])) { + $this->addFlash($type, implode("\n", $allMessages[$type])); + } + } + } } diff --git a/webapp/src/Controller/Jury/ProblemController.php b/webapp/src/Controller/Jury/ProblemController.php index eae7954059..cfd509f905 100644 --- a/webapp/src/Controller/Jury/ProblemController.php +++ b/webapp/src/Controller/Jury/ProblemController.php @@ -972,12 +972,7 @@ public function editAction(Request $request, int $probId): Response $zip->close(); } } - - foreach (['info', 'warning', 'danger'] as $type) { - if (!empty($messages[$type])) { - $this->addFlash($type, implode("\n", $messages[$type])); - } - } + $this->postMessages($messages); return $this->redirectToRoute('jury_problem', ['probId' => $problem->getProbid()]); } @@ -1172,4 +1167,16 @@ public function requestRemainingRunsWholeProblemAction(string $probId): Redirect $this->judgeRemaining($judgings); return $this->redirect($this->generateUrl('jury_problem', ['probId' => $probId])); } + + /** + * @param array $allMessages + */ + private function postMessages(array $allMessages): void + { + foreach (['info', 'warning', 'danger'] as $type) { + if (!empty($allMessages[$type])) { + $this->addFlash($type, implode("\n", $allMessages[$type])); + } + } + } }