diff --git a/judge/judgedaemon.main.php b/judge/judgedaemon.main.php index eb0574fdf0..4fe63f8bc0 100644 --- a/judge/judgedaemon.main.php +++ b/judge/judgedaemon.main.php @@ -6,6 +6,9 @@ * Part of the DOMjudge Programming Contest Jury System and licensed * under the GNU GPL. See README and COPYING for details. */ + +namespace DOMjudge; + if (isset($_SERVER['REMOTE_ADDR'])) { die("Commandline use only"); } @@ -13,770 +16,561 @@ require(ETCDIR . '/judgehost-config.php'); require(LIBDIR . '/lib.misc.php'); -$endpoints = []; -$domjudge_config = []; +define('DONT_CARE', new class {}); -function judging_directory(string $workdirpath, array $judgeTask) : string +class JudgeDaemon { - if (filter_var($judgeTask['submitid'], FILTER_VALIDATE_INT) === false || - filter_var($judgeTask['jobid'], FILTER_VALIDATE_INT) === false) { - error("Malformed data returned in judgeTask IDs: " . var_export($judgeTask, true)); - } + private static ?JudgeDaemon $instance = null; - return $workdirpath . '/' - . $judgeTask['submitid'] . '/' - . $judgeTask['jobid']; -} + private array $endpoints = []; + private array $domjudge_config = []; + private string $myhost; + private int $verbose = LOG_INFO; + private ?string $daemonid = null; + private array $options = []; -function read_credentials(): void -{ - global $endpoints; + private bool $exitsignalled = false; + private bool $gracefulexitsignalled = false; - $credfile = ETCDIR . '/restapi.secret'; - if (!is_readable($credfile)) { - error("REST API credentials file " . $credfile . " is not readable or does not exist."); - } - $credentials = file($credfile); - if ($credentials === false) { - error("Error reading REST API credentials file " . $credfile); - } - $lineno = 0; - foreach ($credentials as $credential) { - ++$lineno; - $credential = trim($credential); - if ($credential === '' || $credential[0] === '#') { - continue; - } - /** @var string[] $items */ - $items = preg_split("/\s+/", $credential); - if (count($items) !== 4) { - error("Error parsing REST API credentials. Invalid format in line $lineno."); - } - [$endpointID, $resturl, $restuser, $restpass] = $items; - if (array_key_exists($endpointID, $endpoints)) { - error("Error parsing REST API credentials. Duplicate endpoint ID '$endpointID' in line $lineno."); - } - $endpoints[$endpointID] = [ - "url" => $resturl, - "user" => $restuser, - "pass" => $restpass, - "waiting" => false, - "errorred" => false, - "last_attempt" => -1, - "retrying" => false, - ]; - } - if (count($endpoints) <= 0) { - error("Error parsing REST API credentials: no endpoints found."); - } -} + private ?string $lastrequest = ''; + private float $waittime = self::INITIAL_WAITTIME_SEC; -function setup_curl_handle(string $restuser, string $restpass): CurlHandle|false -{ - $curl_handle = curl_init(); - curl_setopt($curl_handle, CURLOPT_USERAGENT, "DOMjudge/" . DOMJUDGE_VERSION); - curl_setopt($curl_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); - curl_setopt($curl_handle, CURLOPT_USERPWD, $restuser . ":" . $restpass); - curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, true); - return $curl_handle; -} + private ?string $endpointID = null; -function close_curl_handles(): void -{ - global $endpoints; - foreach ($endpoints as $id => $endpoint) { - if (! empty($endpoint['ch'])) { - curl_close($endpoint['ch']); - unset($endpoints[$id]['ch']); - } - } -} + private array $langexts = []; -// $lastrequest is used to avoid spamming the log with irrelevant log messages. -$lastrequest = ''; + private $lockfile; + private array $EXITCODES; -/** - * Perform a request to the REST API and handle any errors. - * - * $url is the part appended to the base DOMjudge $resturl. - * $verb is the HTTP method to use: GET, POST, PUT, or DELETE - * $data is the urlencoded data passed as GET or POST parameters. - * - * When $failonerror is set to false, any error will be turned into a - * warning and null is returned. - * - * This function retries requests on transient network errors. - * To deal with the transient errors while avoiding overloads, - * this function uses exponential backoff algorithm with jitter. - * - * Every error except authentication failures (HTTP 401) is - * considered transient, even internal server errors (HTTP 5xx). - */ -function request(string $url, string $verb = 'GET', $data = '', bool $failonerror = true) -{ - global $endpoints, $endpointID, $lastrequest; + const INITIAL_WAITTIME_SEC = 0.1; + const MAXIMAL_WAITTIME_SEC = 5.0; - // Don't flood the log with requests for new judgings every few seconds. - if (str_starts_with($url, 'judgehosts/fetch-work') && $verb==='POST') { - if ($lastrequest!==$url) { - logmsg(LOG_DEBUG, "API request $verb $url"); - $lastrequest = $url; + const SCRIPT_ID = 'judgedaemon'; + const CHROOT_SCRIPT = 'chroot-startstop.sh'; + + public static function signalHandler(int $signal): void + { + if (self::$instance) { + self::$instance->handleSignal($signal); } - } else { - logmsg(LOG_DEBUG, "API request $verb $url"); - $lastrequest = $url; } - $url = $endpoints[$endpointID]['url'] . "/" . $url; - $curl_handle = $endpoints[$endpointID]['ch']; - if ($verb == 'GET') { - $url .= '?' . $data; + public function handleSignal(int $signal): void + { + logmsg(LOG_NOTICE, "Signal $signal received."); + if ($signal === SIGHUP) { + logmsg(LOG_NOTICE, "SIGHUP received, restarting."); + $this->exitsignalled = true; + } elseif ($signal === SIGUSR1) { + $this->gracefulexitsignalled = true; + logmsg(LOG_NOTICE, "SIGUSR1 received, finishing current judging and exiting."); + } else { + $this->exitsignalled = true; + logmsg(LOG_NOTICE, "Received signal, exiting."); + } } - curl_setopt($curl_handle, CURLOPT_URL, $url); + public function __construct() + { + self::$instance = $this; - curl_setopt($curl_handle, CURLOPT_CUSTOMREQUEST, $verb); - curl_setopt($curl_handle, CURLOPT_HTTPHEADER, []); - if ($verb == 'POST') { - curl_setopt($curl_handle, CURLOPT_POST, true); - if (is_array($data)) { - curl_setopt($curl_handle, CURLOPT_HTTPHEADER, ['Content-Type: multipart/form-data']); + $this->options = getopt("dv:n:hV", ["diskspace-error"]); + if ($this->options === false) { + echo "Error: parsing options failed.\n"; + $this->usage(); + } + if (isset($this->options['v'])) { + $this->options['verbose'] = $this->options['v']; + } + if (isset($this->options['n'])) { + $this->options['daemonid'] = $this->options['n']; } - } else { - curl_setopt($curl_handle, CURLOPT_POST, false); - } - if ($verb == 'POST' || $verb == 'PUT') { - curl_setopt($curl_handle, CURLOPT_POSTFIELDS, $data); - } else { - curl_setopt($curl_handle, CURLOPT_POSTFIELDS, null); - } - $delay_in_sec = BACKOFF_INITIAL_DELAY_SEC; - $succeeded = false; - $response = null; - $errstr = null; + if (isset($this->options['V'])) { + $this->version(); + } + if (isset($this->options['h'])) { + $this->usage(); + } - for ($trial = 1; $trial <= BACKOFF_STEPS; $trial++) { - $response = curl_exec($curl_handle); - if ($response === false) { - $errstr = "Error while executing curl $verb to url " . $url . ": " . curl_error($curl_handle); - } else { - $status = curl_getinfo($curl_handle, CURLINFO_HTTP_CODE); - if ($status == 401) { - $errstr = "Authentication failed (error $status) while contacting $url. " . - "Check credentials in restapi.secret."; - // Do not retry on authentication failures. - break; - } elseif ($status < 200 || $status >= 300) { - $json = dj_json_try_decode($response); - if ($json !== null) { - $response = var_export($json, true); + if (posix_getuid() == 0 || posix_geteuid() == 0) { + echo "This program should not be run as root.\n"; + exit(1); + } + + $hostname = gethostname(); + if ($hostname === false) { + error("Could not determine hostname."); + } + $this->myhost = explode('.', $hostname)[0]; + if (isset($this->options['daemonid'])) { + if (preg_match('/^\d+$/', $this->options['daemonid'])) { + $this->myhost = $this->myhost . "-" . $this->options['daemonid']; + $this->daemonid = $this->options['daemonid']; + } else { + echo "Invalid value for daemonid, must be positive integer.\n"; + exit(1); + } + } + + define('LOGFILE', LOGDIR . '/judge.' . $this->myhost . '.log'); + // We can only load this here after defining the LOGFILE. + require(LIBDIR . '/lib.error.php'); + + if (isset($this->options['verbose'])) { + if (preg_match('/^\d+$/', $this->options['verbose'])) { + $this->verbose = (int)$this->options['verbose']; + if ($this->verbose >= LOG_DEBUG) { + // Also enable judging scripts debug output + putenv('DEBUG=1'); } - $errstr = "Error while executing curl $verb to url " . $url . - ": http status code: " . $status . - ", request size = " . strlen(print_r($data, true)) . - ", response: " . $response; } else { - $succeeded = true; - break; + error("Invalid value for verbose, must be positive integer."); } } - if ($trial == BACKOFF_STEPS) { - $errstr = $errstr . " Retry limit reached."; - } else { - $retry_in_sec = $delay_in_sec + BACKOFF_JITTER_SEC*random_int(0, mt_getrandmax())/mt_getrandmax(); - $warnstr = $errstr . " This request will be retried after about " . - $retry_in_sec . "sec... (" . $trial . "/" . BACKOFF_STEPS . ")"; - warning($warnstr); - dj_sleep($retry_in_sec); - $delay_in_sec = $delay_in_sec * BACKOFF_FACTOR; + + global $verbose; + $verbose = $this->verbose; + + $runuser = RUNUSER; + if (isset($this->options['daemonid'])) { + $runuser .= '-' . $this->options['daemonid']; } - } - if (!$succeeded) { - if ($failonerror) { - error($errstr); - } else { - warning($errstr); - $endpoints[$endpointID]['errorred'] = true; - return null; + + if ($runuser === posix_getpwuid(posix_geteuid())['name'] || + RUNGROUP === posix_getgrgid(posix_getegid())['name'] + ) { + error("Do not run the judgedaemon as the runuser or rungroup."); } - } - if ($endpoints[$endpointID]['errorred']) { - $endpoints[$endpointID]['errorred'] = false; - $endpoints[$endpointID]['waiting'] = false; - logmsg(LOG_NOTICE, "Reconnected to endpoint $endpointID."); - } + // Set static environment variables for passing path configuration + // to called programs: + putenv('DJ_BINDIR=' . BINDIR); + putenv('DJ_ETCDIR=' . ETCDIR); + putenv('DJ_JUDGEDIR=' . JUDGEDIR); + putenv('DJ_LIBDIR=' . LIBDIR); + putenv('DJ_LIBJUDGEDIR=' . LIBJUDGEDIR); + putenv('DJ_LOGDIR=' . LOGDIR); + putenv('RUNUSER=' . $runuser); + putenv('RUNGROUP=' . RUNGROUP); + + global $EXITCODES; + $this->EXITCODES = $EXITCODES; + foreach ($this->EXITCODES as $code => $name) { + $var = 'E_' . strtoupper(str_replace('-', '_', $name)); + putenv($var . '=' . $code); + } - return $response; -} + // Pass SYSLOG variable via environment for compare program + if (defined('SYSLOG') && SYSLOG) { + putenv('DJ_SYSLOG=' . SYSLOG); + } -/** - * Retrieve the configuration through the REST API. - */ -function djconfig_refresh(): void -{ - global $domjudge_config; + // The judgedaemon calls itself to send judging results back to the API + // asynchronously. See the handling of the 'e' option below. The code here + // should only be run during a normal judgedaemon start. + if (empty($this->options['e'])) { + if (!posix_getpwnam($runuser)) { + error("runuser $runuser does not exist."); + } - $res = request('config', 'GET'); - $res = dj_json_decode($res); - $domjudge_config = $res; -} + define('LOCKFILE', RUNDIR . '/judge.' . $this->myhost . '.lock'); + if (($this->lockfile = fopen(LOCKFILE, 'c')) === false) { + error("cannot open lockfile '" . LOCKFILE . "' for writing"); + } + if (!flock($this->lockfile, LOCK_EX | LOCK_NB)) { + error("cannot lock '" . LOCKFILE . "', is another judgedaemon already running?"); + } + if (!ftruncate($this->lockfile, 0) || fwrite($this->lockfile, (string)getmypid()) === false) { + error("cannot write PID to '" . LOCKFILE . "'"); + } -/** - * Retrieve a specific value from the DOMjudge configuration. - */ -function djconfig_get_value(string $name) -{ - global $domjudge_config; - if (empty($domjudge_config)) { - djconfig_refresh(); - } + $output = []; + exec("ps -u '$runuser' -o pid= -o comm=", $output, $retval); + if (count($output) !== 0) { + error("found processes still running as '$runuser', check manually:\n" . + implode("\n", $output)); + } + + logmsg(LOG_NOTICE, "Judge started on $this->myhost [DOMjudge/" . DOMJUDGE_VERSION . "]"); + } + + $this->initsignals(); - if (!array_key_exists($name, $domjudge_config)) { - error("Configuration value '$name' not found in config."); + $this->readCredentials(); } - return $domjudge_config[$name]; -} -/** - * Encode file contents for POST-ing to REST API. - * - * Returns contents of $file (optionally limited in size, see - * dj_file_get_contents) as encoded string. - * - * $sizelimit can be set to the following values: - * - TRUE: use the 'output_storage_limit' configuration setting - * - positive integer: limit to this many bytes - * - FALSE or -1: no size limit imposed - */ -function rest_encode_file(string $file, $sizelimit = true) : string -{ - $maxsize = null; - if ($sizelimit===true) { - $maxsize = (int) djconfig_get_value('output_storage_limit'); - } elseif ($sizelimit===false || $sizelimit==-1) { - $maxsize = -1; - } elseif (is_int($sizelimit) && $sizelimit>0) { - $maxsize = $sizelimit; - } else { - error("Invalid argument sizelimit = '$sizelimit' specified."); + public function run(): void + { + $this->initialize(); + + // Constantly check API for outstanding judgetasks, cycling through all + // configured endpoints. + $this->loop(); } - return base64_encode(dj_file_get_contents($file, $maxsize)); -} -const INITIAL_WAITTIME_SEC = 0.1; -const MAXIMAL_WAITTIME_SEC = 5.0; -$waittime = INITIAL_WAITTIME_SEC; + private function initialize(): void + { + // Set umask to allow group and other access, as this is needed for the + // unprivileged user. + umask(0022); -const SCRIPT_ID = 'judgedaemon'; -const CHROOT_SCRIPT = 'chroot-startstop.sh'; + // Check basic prerequisites for chroot at judgehost startup + logmsg(LOG_INFO, "🔏 Executing chroot script: '" . self::CHROOT_SCRIPT . " check'"); + if (!$this->run_command_safe([LIBJUDGEDIR . '/' . self::CHROOT_SCRIPT, 'check'])) { + error("chroot validation check failed"); + } -function usage(): never -{ - echo "Usage: " . SCRIPT_ID . " [OPTION]...\n" . - "Start the judgedaemon.\n\n" . - " -n bind to CPU and user " . RUNUSER . "-\n" . - " --diskspace-error send internal error on low diskspace; if not set,\n" . - " the judgedaemon will try to clean up and continue\n" . - " -v set verbosity to ; these are syslog levels:\n" . - " default is LOG_INFO = 5, max is LOG_DEBUG = 7\n" . - " -h display this help and exit\n" . - " -V output version information and exit\n\n"; - exit; -} + foreach (array_keys($this->endpoints) as $id) { + $this->endpointID = $id; + $this->registerJudgehost(); + } -function read_judgehostlog(int $numLines = 20) : string -{ - ob_start(); - passthru("tail -n $numLines " . dj_escapeshellarg(LOGFILE)); - return trim(ob_get_clean()); -} + // Populate the DOMjudge configuration initially + $this->djconfig_refresh(); -define('DONT_CARE', new class {}); -/** - * Execute a shell command given an array of strings forming the command. - * The command and all its arguments are automatically escaped. - * - * @param array $command_parts The command and its arguments (e.g., ['ls', '-l', '/tmp/']). - * @param mixed $retval The (integer) variable to store the command's exit code. - * @param bool $log_nonzero_exitcode Whether non-zero exit codes should be logged. - * - * @return bool Returns true on success (exit code 0), false otherwise. - */ -function run_command_safe(array $command_parts, &$retval = DONT_CARE, $log_nonzero_exitcode = true): bool -{ - if (empty($command_parts)) { - logmsg(LOG_WARNING, "Need at least the command that should be called."); - $retval = -1; - return false; + // Prepopulate default language extensions, afterwards update based on + // domserver config. + $this->langexts = [ + 'c' => ['c'], + 'cpp' => ['cpp', 'C', 'cc'], + 'java' => ['java'], + 'py' => ['py'], + ]; + $domserver_languages = dj_json_decode($this->request('languages', 'GET')); + foreach ($domserver_languages as $language) { + $id = $language['id']; + if (key_exists($id, $this->langexts)) { + $this->langexts[$id] = $language['extensions']; + } + } } - $command = implode(' ', array_map('dj_escapeshellarg', $command_parts)); + private function loop(): void + { + $endpointIDs = array_keys($this->endpoints); + $currentEndpoint = 0; + $lastWorkdir = null; + while (true) { + // If all endpoints are waiting, sleep for a bit. + $dosleep = true; + foreach ($this->endpoints as $id => $endpoint) { + if ($endpoint['errorred']) { + $this->endpointID = $id; + $this->registerJudgehost(); + } + if (!$endpoint['waiting']) { + $dosleep = false; + $this->waittime = self::INITIAL_WAITTIME_SEC; + break; + } + } + // Sleep only if everything is "waiting" and only if we're looking at the first endpoint again. + if ($dosleep && $currentEndpoint == 0) { + dj_sleep($this->waittime); + $this->waittime = min($this->waittime * 2, self::MAXIMAL_WAITTIME_SEC); + } - logmsg(LOG_DEBUG, "Executing command: $command"); - system($command, $retval_local); - if ($retval !== DONT_CARE) $retval = $retval_local; + // Cycle through endpoints. + $currentEndpoint = ($currentEndpoint + 1) % count($this->endpoints); + $this->endpointID = $endpointIDs[$currentEndpoint]; + $workdirpath = JUDGEDIR . "/$this->myhost/endpoint-$this->endpointID"; - if ($retval_local !== 0) { - if ($log_nonzero_exitcode) { - logmsg(LOG_WARNING, "Command failed with exit code $retval_local: $command"); - } - return false; - } + // Check whether we have received an exit signal + if (function_exists('pcntl_signal_dispatch')) { + pcntl_signal_dispatch(); + } + if (function_exists('pcntl_waitpid')) { + // Reap any finished child processes. + while (pcntl_waitpid(-1, $status, WNOHANG) > 0) { + // Do nothing. + } + } + if ($this->exitsignalled) { + logmsg(LOG_NOTICE, "Received signal, exiting."); + $this->close_curl_handles(); + fclose($this->lockfile); + exit; + } - return true; -} + if ($this->endpoints[$this->endpointID]['errorred']) { + continue; + } -// Fetches a new executable from database if not cached already, and runs build script to compile executable. -// Returns an array with -// - absolute path to run script -// - optional error message. -function fetch_executable( - string $workdirpath, - string $type, - string $execid, - string $hash, - int $judgeTaskId, - bool $combined_run_compare = false -) : array { - [$execrunpath, $error, $buildlogpath] = fetch_executable_internal($workdirpath, $type, $execid, $hash, $combined_run_compare); - if (isset($error)) { - $extra_log = null; - if ($buildlogpath !== null) { - $extra_log = dj_file_get_contents($buildlogpath, 4096); - } - logmsg(LOG_ERR, - "Fetching executable failed for $type script '$execid': " . $error); - $description = "$execid: fetch, compile, or deploy of $type script failed."; - disable( - $type . '_script', - $type . '_script_id', - $execid, - $description, - $judgeTaskId, - $extra_log - ); - } - return [$execrunpath, $error]; -} -// Internal function to fetch a new executable from database if necessary, and run build script to compile executable. -// Returns an array with -// - absolute path to run script (null if unsuccessful) -// - an error message (null if successful) -// - optional extra build log. -function fetch_executable_internal( - string $workdirpath, - string $type, - string $execid, - string $hash, - bool $combined_run_compare = false -) : array { - $execdir = join('/', [ - $workdirpath, - 'executable', - $type, - $execid, - $hash - ]); - global $langexts; - global $myhost; - $execdeploypath = $execdir . '/.deployed'; - $execbuilddir = $execdir . '/build'; - $execbuildpath = $execbuilddir . '/build'; - $execrunpath = $execbuilddir . '/run'; - $execrunjurypath = $execbuilddir . '/runjury'; - if (!is_dir($execdir) || !file_exists($execdeploypath) || - ($combined_run_compare && file_get_contents(LIBJUDGEDIR . '/run-interactive.sh')!==file_get_contents($execrunpath))) { - if (!run_command_safe(['rm', '-rf', $execdir, $execbuilddir])) { - disable('judgehost', 'hostname', $myhost, "Deleting '$execdir' or '$execbuilddir' was unsuccessful."); - } - if (!run_command_safe(['mkdir', '-p', $execbuilddir])) { - disable('judgehost', 'hostname', $myhost, "Could not create directory '$execbuilddir'"); - } - - logmsg(LOG_INFO, " 💾 Fetching new executable '$type/$execid' with hash '$hash'."); - $content = request(sprintf('judgehosts/get_files/%s/%s', $type, $execid), 'GET'); - $files = dj_json_decode($content); - unset($content); - $filesArray = []; - foreach ($files as $file) { - $filename = $execbuilddir . '/' . $file['filename']; - $content = base64_decode($file['content']); - file_put_contents($filename, $content); - if ($file['is_executable']) { - chmod($filename, 0755); - } - $filesArray[] = [ - 'hash' => md5($content), - 'filename' => $file['filename'], - 'is_executable' => $file['is_executable'], - ]; - } - unset($files); - uasort($filesArray, fn(array $a, array $b) => strcmp($a['filename'], $b['filename'])); - $computedHash = md5( - join( - array_map( - fn($file) => $file['hash'] . $file['filename'] . $file['is_executable'], - $filesArray - ) - ) - ); - if ($hash !== $computedHash) { - return [null, "Unexpected hash ($computedHash), expected hash: $hash", null]; - } + if ($this->endpoints[$this->endpointID]['waiting'] === false) { + $this->checkDiskSpace($workdirpath); + } + + // Request open judge tasks to be executed. + // Any errors will be treated as non-fatal: we will just keep on retrying in this loop. + $row = $this->fetchWork(); - $do_compile = true; - if (!file_exists($execbuildpath)) { - if (file_exists($execrunpath)) { - // 'run' already exists, 'build' does not => don't compile anything - logmsg(LOG_DEBUG, "'run' exists without 'build', we are done."); - $do_compile = false; + // If $judging is null, an error occurred; we marked the endpoint already as errorred above. + if (is_null($row)) { + continue; } else { - // detect lang and write build file - $buildscript = "#!/bin/sh\n\n"; - $execlang = false; - $source = ""; - $unescapedSource = ""; - foreach ($langexts as $lang => $langext) { - if (($handle = opendir($execbuilddir)) === false) { - disable('judgehost', 'hostname', $myhost, "Could not open $execbuilddir"); + $row = dj_json_decode($row); + } + + // Nothing returned -> no open work for us. + if (empty($row)) { + if (!$this->endpoints[$this->endpointID]["waiting"]) { + $this->endpoints[$this->endpointID]["waiting"] = true; + if ($lastWorkdir !== null) { + $this->cleanup_judging($lastWorkdir); + $lastWorkdir = null; } - while (($file = readdir($handle)) !== false) { - $ext = pathinfo($file, PATHINFO_EXTENSION); - if (in_array($ext, $langext)) { - $execlang = $lang; - $unescapedSource = $file; - $source = dj_escapeshellarg($unescapedSource); - break; + logmsg(LOG_INFO, "No submissions in queue (for endpoint $this->endpointID), waiting..."); + $judgehosts = $this->request('judgehosts', 'GET'); + if ($judgehosts !== null) { + $judgehosts = dj_json_decode($judgehosts); + $judgehost = array_values(array_filter($judgehosts, fn($j) => $j['hostname'] === $this->myhost))[0]; + if (!isset($judgehost['enabled']) || !$judgehost['enabled']) { + logmsg(LOG_WARNING, "Judgehost needs to be enabled in web interface."); } } - closedir($handle); - if ($execlang !== false) { - break; - } - } - if ($execlang === false) { - return [null, "executable must either provide an executable file named 'build' or a C/C++/Java or Python file.", null]; - } - switch ($execlang) { - case 'c': - $buildscript .= "gcc -Wall -O2 -std=gnu11 $source -o run -lm\n"; - break; - case 'cpp': - $buildscript .= "g++ -Wall -O2 -std=gnu++20 $source -o run\n"; - break; - case 'java': - $buildscript .= "javac -cp . -d . $source\n"; - $buildscript .= "echo '#!/bin/sh' > run\n"; - // no main class detection here - $buildscript .= "echo 'COMPARE_DIR=\$(dirname \"\$0\")' >> run\n"; - $mainClass = basename($unescapedSource, '.java'); - // Note: since the $@ is within single quotes, we do not need to double escape it. - $buildscript .= "echo 'java -cp \"\$COMPARE_DIR\" $mainClass \"\$@\"' >> run\n"; - $buildscript .= "chmod +x run\n"; - break; - case 'py': - $buildscript .= "echo '#!/bin/sh' > run\n"; - $buildscript .= "echo 'COMPARE_DIR=\$(dirname \"\$0\")' >> run\n"; - // Note: since the $@ is within single quotes, we do not need to double escape it. - $buildscript .= "echo 'python3 \"\$COMPARE_DIR/$source\" \"\$@\"' >> run\n"; - $buildscript .= "chmod +x run\n"; - break; - } - if (file_put_contents($execbuildpath, $buildscript) === false) { - disable('judgehost', 'hostname', $myhost, "Could not write file 'build' in $execbuilddir"); } - chmod($execbuildpath, 0755); + continue; } - } elseif (!is_executable($execbuildpath)) { - return [null, "Invalid executable, file 'build' exists but is not executable.", null]; - } - - if ($do_compile) { - logmsg(LOG_DEBUG, "Building executable in $execdir, under 'build/'"); - putenv('SCRIPTTIMELIMIT=' . djconfig_get_value('script_timelimit')); - putenv('SCRIPTMEMLIMIT=' . djconfig_get_value('script_memory_limit')); - putenv('SCRIPTFILELIMIT=' . djconfig_get_value('script_filesize_limit')); + // We have gotten a work packet. + $this->endpoints[$this->endpointID]["waiting"] = false; - if (!run_command_safe([LIBJUDGEDIR . '/build_executable.sh', $execdir])) { - return [null, "Failed to build executable in $execdir.", "$execdir/build.log"]; + // All tasks are guaranteed to be of the same type. + // If $row is empty, we already continued. + // If $row[0] is not set, or $row[0]['type'] is not set, something is wrong. + if (!isset($row[0]['type'])) { + logmsg(LOG_ERR, "Received work packet with invalid format: 'type' not found in first element."); + continue; } - chmod($execrunpath, 0755); + $type = $row[0]['type']; + + $this->handleTask($type, $row, $lastWorkdir, $workdirpath); } - if (!is_file($execrunpath) || !is_executable($execrunpath)) { - return [null, "Invalid build file, must produce an executable file 'run'.", null]; + } + + private function handleJudgingTask(array $row, ?string &$lastWorkdir, string $workdirpath, string $workdir): void + { + $success_file = "$workdir/.uuid_pid"; + $expected_uuid_pid = $row[0]['uuid'] . '_' . (string)getmypid(); + + $needs_cleanup = false; + if ($lastWorkdir !== $workdir) { + // Switching between workdirs requires cleanup. + $needs_cleanup = true; } - if ($combined_run_compare) { - # For combined run and compare (i.e. for interactive problems), we - # need to wrap the jury provided 'run' script with 'runpipe' to - # handle the bidirectional communication. First 'run' is renamed to - # 'runjury', and then replaced by the script below, which runs the - # team submission and runjury programs and connects their pipes. - $runscript = file_get_contents(LIBJUDGEDIR . '/run-interactive.sh'); - if (rename($execrunpath, $execrunjurypath) === false) { - disable('judgehost', 'hostname', $myhost, "Could not move file 'run' to 'runjury' in $execbuilddir"); + if (file_exists($workdir)) { + // If the workdir still exists we need to check whether it may be a left-over from a previous database. + // If that is the case, we need to rename it and potentially clean up. + if (file_exists($success_file)) { + $old_uuid_pid = file_get_contents($success_file); + if ($old_uuid_pid !== $expected_uuid_pid) { + $needs_cleanup = true; + unlink($success_file); + } + } else { + $old_uuid_pid = 'n/a'; + $needs_cleanup = true; } - if (file_put_contents($execrunpath, $runscript) === false) { - disable('judgehost', 'hostname', $myhost, "Could not write file 'run' in $execbuilddir"); + + // Either the file didn't exist or we deleted it above. + if (!file_exists($success_file)) { + $oldworkdir = $workdir . '-old-' . getmypid() . '-' . date('Y-m-d_H:i'); + if (!rename($workdir, $oldworkdir)) { + error("Could not rename stale working directory to '$oldworkdir'."); + } + @chmod($oldworkdir, 0700); + warning("Found stale working directory; renamed to '$oldworkdir'."); } - chmod($execrunpath, 0755); } - if (!is_file($execrunpath) || !is_executable($execrunpath)) { - return [null, "Invalid build file, must produce an executable file 'run'.", null]; + if ($needs_cleanup && $lastWorkdir !== null) { + $this->cleanup_judging($lastWorkdir); + $lastWorkdir = null; } - // Create file to mark executable successfully deployed. - touch($execdeploypath); - } - - return [$execrunpath, null, null]; -} -$options = getopt("dv:n:hVe:j:t:", ["diskspace-error"]); -// We can't fully trust the output of getopt, it has outstanding bugs: -// https://bugs.php.net/search.php?cmd=display&search_for=getopt&x=0&y=0 -if ($options===false) { - echo "Error: parsing options failed.\n"; - usage(); -} -if (isset($options['v'])) { - $options['verbose'] = $options['v']; -} -if (isset($options['n'])) { - $options['daemonid'] = $options['n']; -} + if (!$this->run_command_safe(['mkdir', '-p', "$workdir/compile"])) { + error("Could not create '$workdir/compile'"); + } -if (isset($options['V'])) { - version(); -} -if (isset($options['h'])) { - usage(); -} + chmod($workdir, 0755); -if (posix_getuid()==0 || posix_geteuid()==0) { - echo "This program should not be run as root.\n"; - exit(1); -} + if (!chdir($workdir)) { + error("Could not chdir to '$workdir'"); + } -$hostname = gethostname(); -if ($hostname === false) { - error("Could not determine hostname."); -} -$myhost = explode('.', $hostname)[0]; -if (isset($options['daemonid'])) { - if (preg_match('/^\d+$/', $options['daemonid'])) { - $myhost = $myhost . "-" . $options['daemonid']; - } else { - echo "Invalid value for daemonid, must be positive integer.\n"; - exit(1); - } -} + if ($lastWorkdir !== $workdir) { + // create chroot environment + logmsg(LOG_INFO, " 🔒 Executing chroot script: '" . self::CHROOT_SCRIPT . " start'"); + if (!$this->run_command_safe([LIBJUDGEDIR . '/' . self::CHROOT_SCRIPT, 'start'], $retval)) { + logmsg(LOG_ERR, "chroot script exited with exitcode $retval"); + $this->disable('judgehost', 'hostname', $this->myhost, "chroot script exited with exitcode $retval on $this->myhost"); + return; + } -define('LOGFILE', LOGDIR.'/judge.'.$myhost.'.log'); -require(LIBDIR . '/lib.error.php'); + // Refresh config at start of each batch. + $this->djconfig_refresh(); -$verbose = LOG_INFO; -if (isset($options['verbose'])) { - if (preg_match('/^\d+$/', $options['verbose'])) { - $verbose = $options['verbose']; - if ($verbose >= LOG_DEBUG) { - // Also enable judging scripts debug output - putenv('DEBUG=1'); + $lastWorkdir = $workdir; } - } else { - error("Invalid value for verbose, must be positive integer."); - } -} - -$runuser = RUNUSER; -if (isset($options['daemonid'])) { - $runuser .= '-' . $options['daemonid']; -} - -if ($runuser === posix_getpwuid(posix_geteuid())['name'] || - RUNGROUP === posix_getgrgid(posix_getegid())['name'] -) { - error("Do not run the judgedaemon as the runuser or rungroup."); -} -// Set static environment variables for passing path configuration -// to called programs: -putenv('DJ_BINDIR=' . BINDIR); -putenv('DJ_ETCDIR=' . ETCDIR); -putenv('DJ_JUDGEDIR=' . JUDGEDIR); -putenv('DJ_LIBDIR=' . LIBDIR); -putenv('DJ_LIBJUDGEDIR=' . LIBJUDGEDIR); -putenv('DJ_LOGDIR=' . LOGDIR); -putenv('RUNUSER=' . $runuser); -putenv('RUNGROUP=' . RUNGROUP); - -foreach ($EXITCODES as $code => $name) { - $var = 'E_' . strtoupper(str_replace('-', '_', $name)); - putenv($var . '=' . $code); -} + // Make sure the workdir is accessible for the domjudge-run user. + // Will be revoked again after this run finished. + foreach ($row as $judgetask) { + if (!$this->compile_and_run_submission($judgetask, $workdirpath)) { + // Potentially return remaining outstanding judgetasks here. + $returnedJudgings = $this->request('judgehosts', 'POST', ['hostname' => urlencode($this->myhost)], false); + if ($returnedJudgings !== null) { + $returnedJudgings = dj_json_decode($returnedJudgings); + foreach ($returnedJudgings as $jud) { + $workdir = $this->judging_directory($workdirpath, $jud); + @chmod($workdir, 0700); + logmsg(LOG_WARNING, " 🔙 Returned unfinished judging with jobid " . $jud['jobid'] . + " in my name; given back unfinished runs from me."); + } + } + break; + } + } -// Pass SYSLOG variable via environment for compare program -if (defined('SYSLOG') && SYSLOG) { - putenv('DJ_SYSLOG=' . SYSLOG); -} + file_put_contents($success_file, $expected_uuid_pid); -// The judgedaemon calls itself to send judging results back to the API -// asynchronously. See the handling of the 'e' option below. The code here -// should only be run during a normal judgedaemon start. -if (empty($options['e'])) { - if (!posix_getpwnam($runuser)) { - error("runuser $runuser does not exist."); + // Check if we were interrupted while judging, if so, exit (to avoid sleeping) + if ($this->exitsignalled) { + logmsg(LOG_NOTICE, "Received signal, exiting."); + $this->close_curl_handles(); + fclose($this->lockfile); + exit; + } } - define('LOCKFILE', RUNDIR.'/judge.'.$myhost.'.lock'); - if (($lockfile = fopen(LOCKFILE, 'c'))===false) { - error("cannot open lockfile '" . LOCKFILE . "' for writing"); - } - if (!flock($lockfile, LOCK_EX | LOCK_NB)) { - error("cannot lock '" . LOCKFILE . "', is another judgedaemon already running?"); - } - if (!ftruncate($lockfile, 0) || fwrite($lockfile, (string)getmypid())===false) { - error("cannot write PID to '" . LOCKFILE . "'"); - } + private function handleDebugInfoTask(array $row, ?string &$lastWorkdir, string $workdirpath, string $workdir): void + { + if ($lastWorkdir !== null) { + $this->cleanup_judging($lastWorkdir); + $lastWorkdir = null; + } + foreach ($row as $judgeTask) { + if (isset($judgeTask['run_script_id'])) { + // Full debug package requested. + $run_config = dj_json_decode($judgeTask['run_config']); + $tmpfile = tempnam(TMPDIR, 'full_debug_package_'); + [$runpath, $error] = $this->fetch_executable( + $workdirpath, + 'debug', + $judgeTask['run_script_id'], + $run_config['hash'], + $judgeTask['judgetaskid'] + ); - $output = []; - exec("ps -u '$runuser' -o pid= -o comm=", $output, $retval); - if (count($output) !== 0) { - error("found processes still running as '$runuser', check manually:\n" . - implode("\n", $output)); - } + if (!$this->run_command_safe([$runpath, $workdir, $tmpfile])) { + $this->disable('run_script', 'run_script_id', $judgeTask['run_script_id'], "Running '$runpath' failed."); + } - logmsg(LOG_NOTICE, "Judge started on $myhost [DOMjudge/" . DOMJUDGE_VERSION . "]"); -} + $this->request( + sprintf('judgehosts/add-debug-info/%s/%s', urlencode($this->myhost), + urlencode((string)$judgeTask['judgetaskid'])), + 'POST', + ['full_debug' => $this->rest_encode_file($tmpfile, false)], + false + ); + unlink($tmpfile); -initsignals(); - -read_credentials(); - -if (!empty($options['e'])) { - $endpointID = $options['e']; - $endpoint = $endpoints[$endpointID]; - $endpoints[$endpointID]['ch'] = setup_curl_handle($endpoint['user'], $endpoint['pass']); - $new_judging_run = (array) dj_json_decode(base64_decode(file_get_contents($options['j']))); - $judgeTaskId = $options['t']; - - $success = false; - for ($i = 0; $i < 5; $i++) { - if ($i > 0) { - $sleep_ms = 100 + random_int(200, ($i+1)*1000); - dj_sleep(0.001 * $sleep_ms); - } - $response = request( - sprintf('judgehosts/add-judging-run/%s/%s', $new_judging_run['hostname'], - urlencode((string)$judgeTaskId)), - 'POST', - $new_judging_run, - false - ); - if ($response !== null) { - logmsg(LOG_DEBUG, "Adding judging run result for jt$judgeTaskId successful."); - $success = true; - break; + logmsg(LOG_INFO, " ⇡ Uploading debug package of workdir $workdir."); + } else { + // Retrieving full team output for a particular testcase. + $testcasedir = $workdir . "/testcase" . sprintf('%05d', $judgeTask['testcase_id']); + $this->request( + sprintf('judgehosts/add-debug-info/%s/%s', urlencode($this->myhost), + urlencode((string)$judgeTask['judgetaskid'])), + 'POST', + ['output_run' => $this->rest_encode_file($testcasedir . '/program.out', false)], + false + ); + logmsg(LOG_INFO, " ⇡ Uploading full output of testcase $judgeTask[testcase_id]."); + } } - logmsg(LOG_WARNING, "Failed to report jt$judgeTaskId in attempt #" . ($i + 1) . "."); } - if (!$success) { - error("Final attempt of uploading jt$judgeTaskId was unsuccessful, giving up."); + + private function handlePrefetchTask(array $row, ?string &$lastWorkdir, string $workdirpath): void + { + if ($lastWorkdir !== null) { + $this->cleanup_judging($lastWorkdir); + $lastWorkdir = null; + } + foreach ($row as $judgeTask) { + foreach (['compile', 'run', 'compare'] as $script_type) { + if (!empty($judgeTask[$script_type . '_script_id']) && !empty($judgeTask[$script_type . '_config'])) { + $config = dj_json_decode($judgeTask[$script_type . '_config']); + $combined_run_compare = $script_type == 'run' && ($config['combined_run_compare'] ?? false); + if (!empty($config['hash'])) { + [$execrunpath, $error] = $this->fetch_executable( + $workdirpath, + $script_type, + $judgeTask[$script_type . '_script_id'], + $config['hash'], + $judgeTask['judgetaskid'], + $combined_run_compare + ); + } + } + } + if (!empty($judgeTask['testcase_id'])) { + $this->fetchTestcase($workdirpath, $judgeTask['testcase_id'], $judgeTask['judgetaskid'], $judgeTask['testcase_hash']); + } + } + logmsg(LOG_INFO, " 🔥 Pre-heating judgehost completed."); } - unlink($options['j']); - exit(0); -} -// Set umask to allow group and other access, as this is needed for the -// unprivileged user. -umask(0022); + private function handleTask(string $type, array $row, ?string &$lastWorkdir, string $workdirpath): void + { + if ($type == 'try_again') { + if (!$this->endpoints[$this->endpointID]['retrying']) { + logmsg(LOG_INFO, "API indicated to retry fetching work (this might take a while to clean up)."); + } + $this->endpoints[$this->endpointID]['retrying'] = true; + return; + } + $this->endpoints[$this->endpointID]['retrying'] = false; -// Check basic prerequisites for chroot at judgehost startup -logmsg(LOG_INFO, "🔏 Executing chroot script: '".CHROOT_SCRIPT." check'"); -if (!run_command_safe([LIBJUDGEDIR.'/'.CHROOT_SCRIPT, 'check'])) { - error("chroot validation check failed"); -} + logmsg(LOG_INFO, + "⇝ Received " . sizeof($row) . " '" . $type . "' judge tasks (endpoint $this->endpointID)"); -foreach ($endpoints as $id => $endpoint) { - $endpointID = $id; - registerJudgehost($myhost); -} + if ($type == 'prefetch') { + $this->handlePrefetchTask($row, $lastWorkdir, $workdirpath); + return; + } -// Populate the DOMjudge configuration initially -djconfig_refresh(); - -// Prepopulate default language extensions, afterwards update based on domserver config -$langexts = [ - 'c' => ['c'], - 'cpp' => ['cpp', 'C', 'cc'], - 'java' => ['java'], - 'py' => ['py'], -]; -$domserver_languages = dj_json_decode(request('languages', 'GET')); -foreach ($domserver_languages as $language) { - $id = $language['id']; - if (key_exists($id, $langexts)) { - $langexts[$id] = $language['extensions']; - } -} + if ($type == 'debug_info') { + // Create workdir for debugging only if needed. + $workdir = $this->judging_directory($workdirpath, $row[0]); + logmsg(LOG_INFO, " Working directory: $workdir"); -// Constantly check API for outstanding judgetasks, cycling through all configured endpoints. -$endpointIDs = array_keys($endpoints); -$currentEndpoint = 0; -$lastWorkdir = null; -while (true) { - // If all endpoints are waiting, sleep for a bit. - $dosleep = true; - foreach ($endpoints as $id => $endpoint) { - if ($endpoint['errorred']) { - $endpointID = $id; - registerJudgehost($myhost); - } - if (!$endpoint['waiting']) { - $dosleep = false; - $waittime = INITIAL_WAITTIME_SEC; - break; + $this->handleDebugInfoTask($row, $lastWorkdir, $workdirpath, $workdir); + return; } - } - // Sleep only if everything is "waiting" and only if we're looking at the first endpoint again. - if ($dosleep && $currentEndpoint==0) { - dj_sleep($waittime); - $waittime = min($waittime*2, MAXIMAL_WAITTIME_SEC); - } - // Cycle through endpoints. - $currentEndpoint = ($currentEndpoint + 1) % count($endpoints); - $endpointID = $endpointIDs[$currentEndpoint]; - $workdirpath = JUDGEDIR . "/$myhost/endpoint-$endpointID"; - - // Check whether we have received an exit signal - if (function_exists('pcntl_signal_dispatch')) { - pcntl_signal_dispatch(); - } - if ($exitsignalled) { - logmsg(LOG_NOTICE, "Received signal, exiting."); - close_curl_handles(); - fclose($lockfile); - exit; + // Create workdir for judging. + $workdir = $this->judging_directory($workdirpath, $row[0]); + logmsg(LOG_INFO, " Working directory: $workdir"); + $this->handleJudgingTask($row, $lastWorkdir, $workdirpath, $workdir); } - if ($endpoints[$endpointID]['errorred']) { - continue; + private function fetchWork() + { + return $this->request('judgehosts/fetch-work', 'POST', ['hostname' => $this->myhost], false); } - - if ($endpoints[$endpointID]['waiting'] === false) { + private function checkDiskSpace(string $workdirpath): void + { // Check for available disk space $free_space = disk_free_space(JUDGEDIR); - $allowed_free_space = djconfig_get_value('diskspace_error'); // in kB - if ($free_space < 1024*$allowed_free_space) { + $allowed_free_space = $this->djconfig_get_value('diskspace_error'); // in kB + if ($free_space < 1024 * $allowed_free_space) { $after = disk_free_space(JUDGEDIR); - if (!isset($options['diskspace-error'])) { + if (!isset($this->options['diskspace-error'])) { $candidateDirs = []; foreach (scandir($workdirpath) as $subdir) { if (is_numeric($subdir) && is_dir(($workdirpath . "/" . $subdir))) { @@ -791,7 +585,7 @@ function fetch_executable_internal( foreach ($candidateDirs as $d) { $cnt++; logmsg(LOG_INFO, " - deleting $d"); - if (!run_command_safe(['rm', '-rf', $d])) { + if (!$this->run_command_safe(['rm', '-rf', $d])) { logmsg(LOG_WARNING, "Deleting '$d' was unsuccessful."); } $after = disk_free_space(JUDGEDIR); @@ -803,865 +597,1171 @@ function fetch_executable_internal( sprintf("%01.2fMB.", ($after - $before) / (1024 * 1024)) ); } - if ($after < 1024*$allowed_free_space) { - $free_abs = sprintf("%01.2fGB", $after / (1024*1024*1024)); + if ($after < 1024 * $allowed_free_space) { + $free_abs = sprintf("%01.2fGB", $after / (1024 * 1024 * 1024)); logmsg(LOG_ERR, "Low on disk space: $free_abs free, clean up or " . "change 'diskspace error' value in config before resolving this error."); - disable('judgehost', 'hostname', $myhost, "low on disk space on $myhost"); + $this->disable('judgehost', 'hostname', $this->myhost, "low on disk space on $this->myhost"); } } } - // Request open judge tasks to be executed. - // Any errors will be treated as non-fatal: we will just keep on retrying in this loop. - $row = []; - $judging = request('judgehosts/fetch-work', 'POST', ['hostname' => $myhost], false); - // If $judging is null, an error occurred; we marked the endpoint already as errorred above. - if (is_null($judging)) { - continue; - } else { - $row = dj_json_decode($judging); + private function judging_directory(string $workdirpath, array $judgeTask): string + { + if (filter_var($judgeTask['submitid'], FILTER_VALIDATE_INT) === false || + filter_var($judgeTask['jobid'], FILTER_VALIDATE_INT) === false) { + error("Malformed data returned in judgeTask IDs: " . var_export($judgeTask, true)); + } + + return $workdirpath . '/' + . $judgeTask['submitid'] . '/' + . $judgeTask['jobid']; } - // Nothing returned -> no open work for us. - if (empty($row)) { - if (! $endpoints[$endpointID]["waiting"]) { - $endpoints[$endpointID]["waiting"] = true; - if ($lastWorkdir !== null) { - cleanup_judging($lastWorkdir); - $lastWorkdir = null; - } - logmsg(LOG_INFO, "No submissions in queue (for endpoint $endpointID), waiting..."); - $judgehosts = request('judgehosts', 'GET'); - if ($judgehosts !== null) { - $judgehosts = dj_json_decode($judgehosts); - $judgehost = array_values(array_filter($judgehosts, fn($j) => $j['hostname'] === $myhost))[0]; - if (!isset($judgehost['enabled']) || !$judgehost['enabled']) { - logmsg(LOG_WARNING, "Judgehost needs to be enabled in web interface."); - } + private function readCredentials(): void + { + $credfile = ETCDIR . '/restapi.secret'; + if (!is_readable($credfile)) { + error("REST API credentials file " . $credfile . " is not readable or does not exist."); + } + $credentials = file($credfile); + if ($credentials === false) { + error("Error reading REST API credentials file " . $credfile); + } + $lineno = 0; + foreach ($credentials as $credential) { + ++$lineno; + $credential = trim($credential); + if ($credential === '' || $credential[0] === '#') { + continue; } + /** @var string[] $items */ + $items = preg_split("/\s+/", $credential); + if (count($items) !== 4) { + error("Error parsing REST API credentials. Invalid format in line $lineno."); + } + [$endpointID, $resturl, $restuser, $restpass] = $items; + if (array_key_exists($endpointID, $this->endpoints)) { + error("Error parsing REST API credentials. Duplicate endpoint ID '$endpointID' in line $lineno."); + } + $this->endpoints[$endpointID] = [ + "url" => $resturl, + "user" => $restuser, + "pass" => $restpass, + "waiting" => false, + "errorred" => false, + "last_attempt" => -1, + "retrying" => false, + ]; } - continue; - } - - // We have gotten a work packet. - $endpoints[$endpointID]["waiting"] = false; - - // All tasks are guaranteed to be of the same type. - $type = $row[0]['type']; - - if ($type == 'try_again') { - if (!$endpoints[$endpointID]['retrying']) { - logmsg(LOG_INFO, "API indicated to retry fetching work (this might take a while to clean up)."); + if (count($this->endpoints) <= 0) { + error("Error parsing REST API credentials: no endpoints found."); } - $endpoints[$endpointID]['retrying'] = true; - continue; } - $endpoints[$endpointID]['retrying'] = false; - logmsg(LOG_INFO, - "⇝ Received " . sizeof($row) . " '" . $type . "' judge tasks (endpoint $endpointID)"); + private function setup_curl_handle(string $restuser, string $restpass): \CurlHandle|false + { + $curl_handle = curl_init(); + curl_setopt($curl_handle, CURLOPT_USERAGENT, "DOMjudge/" . DOMJUDGE_VERSION); + curl_setopt($curl_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($curl_handle, CURLOPT_USERPWD, $restuser . ":" . $restpass); + curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, true); + return $curl_handle; + } - if ($type == 'prefetch') { - if ($lastWorkdir !== null) { - cleanup_judging($lastWorkdir); - $lastWorkdir = null; - } - foreach ($row as $judgeTask) { - foreach (['compile', 'run', 'compare'] as $script_type) { - if (!empty($judgeTask[$script_type . '_script_id']) && !empty($judgeTask[$script_type . '_config'])) { - $config = dj_json_decode($judgeTask[$script_type . '_config']); - $combined_run_compare = $script_type == 'run' && $config['combined_run_compare']; - if (!empty($config['hash'])) { - [$execrunpath, $error] = fetch_executable( - $workdirpath, - $script_type, - $judgeTask[$script_type . '_script_id'], - $config['hash'], - $judgeTask['judgetaskid'], - $combined_run_compare - ); - } - } - } - if (!empty($judgeTask['testcase_id'])) { - fetchTestcase($workdirpath, $judgeTask['testcase_id'], $judgeTask['judgetaskid'], $judgeTask['testcase_hash']); + private function close_curl_handles(): void + { + foreach ($this->endpoints as $id => $endpoint) { + if (!empty($endpoint['ch'])) { + curl_close($endpoint['ch']); + unset($this->endpoints[$id]['ch']); } } - logmsg(LOG_INFO, " 🔥 Pre-heating judgehost completed."); - continue; } - // Create workdir for judging. - $workdir = judging_directory($workdirpath, $row[0]); - logmsg(LOG_INFO, " Working directory: $workdir"); - - if ($type == 'debug_info') { - if ($lastWorkdir !== null) { - cleanup_judging($lastWorkdir); - $lastWorkdir = null; + private function request(string $url, string $verb = 'GET', $data = '', bool $failonerror = true) + { + // Don't flood the log with requests for new judgings every few seconds. + if (str_starts_with($url, 'judgehosts/fetch-work') && $verb === 'POST') { + if ($this->lastrequest !== $url) { + logmsg(LOG_DEBUG, "API request $verb $url"); + $this->lastrequest = $url; + } + } else { + logmsg(LOG_DEBUG, "API request $verb $url"); + $this->lastrequest = $url; } - foreach ($row as $judgeTask) { - if (isset($judgeTask['run_script_id'])) { - // Full debug package requested. - $run_config = dj_json_decode($judgeTask['run_config']); - $tmpfile = tempnam(TMPDIR, 'full_debug_package_'); - [$runpath, $error] = fetch_executable( - $workdirpath, - 'debug', - $judgeTask['run_script_id'], - $run_config['hash'], - $judgeTask['judgetaskid'] - ); - if (!run_command_safe([$runpath, $workdir, $tmpfile])) { - disable('run_script', 'run_script_id', $judgeTask['run_script_id'], "Running '$runpath' failed."); - } + $requestUrl = $this->endpoints[$this->endpointID]['url'] . "/" . $url; + $curl_handle = $this->endpoints[$this->endpointID]['ch']; + if ($verb == 'GET') { + $requestUrl .= '?' . $data; + } - request( - sprintf('judgehosts/add-debug-info/%s/%s', urlencode($myhost), - urlencode((string)$judgeTask['judgetaskid'])), - 'POST', - ['full_debug' => rest_encode_file($tmpfile, false)], - false - ); - unlink($tmpfile); + curl_setopt($curl_handle, CURLOPT_URL, $requestUrl); - logmsg(LOG_INFO, " ⇡ Uploading debug package of workdir $workdir."); - } else { - // Retrieving full team output for a particular testcase. - $testcasedir = $workdir . "/testcase" . sprintf('%05d', $judgeTask['testcase_id']); - request( - sprintf('judgehosts/add-debug-info/%s/%s', urlencode($myhost), - urlencode((string)$judgeTask['judgetaskid'])), - 'POST', - ['output_run' => rest_encode_file($testcasedir . '/program.out', false)], - false - ); - logmsg(LOG_INFO, " ⇡ Uploading full output of testcase $judgeTask[testcase_id]."); + curl_setopt($curl_handle, CURLOPT_CUSTOMREQUEST, $verb); + curl_setopt($curl_handle, CURLOPT_HTTPHEADER, []); + if ($verb == 'POST') { + curl_setopt($curl_handle, CURLOPT_POST, true); + if (is_array($data)) { + curl_setopt($curl_handle, CURLOPT_HTTPHEADER, ['Content-Type: multipart/form-data']); } + } else { + curl_setopt($curl_handle, CURLOPT_POST, false); + } + if ($verb == 'POST' || $verb == 'PUT') { + curl_setopt($curl_handle, CURLOPT_POSTFIELDS, $data); + } else { + curl_setopt($curl_handle, CURLOPT_POSTFIELDS, null); } - continue; - } - $success_file = "$workdir/.uuid_pid"; - $expected_uuid_pid = $row[0]['uuid'] . '_' . (string)getmypid(); + $delay_in_sec = BACKOFF_INITIAL_DELAY_SEC; + $succeeded = false; + $response = null; + $errstr = null; - $needs_cleanup = false; - if ($lastWorkdir !== $workdir) { - // Switching between workdirs requires cleanup. - $needs_cleanup = true; - } - if (file_exists($workdir)) { - // If the workdir still exists we need to check whether it may be a left-over from a previous database. - // If that is the case, we need to rename it and potentially clean up. - if (file_exists($success_file)) { - $old_uuid_pid = file_get_contents($success_file); - if ($old_uuid_pid !== $expected_uuid_pid) { - $needs_cleanup = true; - unlink($success_file); + for ($trial = 1; $trial <= BACKOFF_STEPS; $trial++) { + $response = curl_exec($curl_handle); + if ($response === false) { + $errstr = "Error while executing curl $verb to url " . $requestUrl . ": " . curl_error($curl_handle); + } else { + $status = curl_getinfo($curl_handle, CURLINFO_HTTP_CODE); + if ($status == 401) { + $errstr = "Authentication failed (error $status) while contacting $requestUrl. " . + "Check credentials in restapi.secret."; + // Do not retry on authentication failures. + break; + } elseif ($status < 200 || $status >= 300) { + $json = dj_json_try_decode($response); + if ($json !== null) { + $response = var_export($json, true); + } + $errstr = "Error while executing curl $verb to url " . $requestUrl . + ": http status code: " . $status . + ", request size = " . strlen(print_r($data, true)) . + ", response: " . $response; + } else { + $succeeded = true; + break; + } + } + if ($trial == BACKOFF_STEPS) { + $errstr = $errstr . " Retry limit reached."; + } else { + $retry_in_sec = $delay_in_sec + BACKOFF_JITTER_SEC * random_int(0, mt_getrandmax()) / mt_getrandmax(); + $warnstr = $errstr . " This request will be retried after about " . + round($retry_in_sec, 2) . "sec... (" . $trial . "/" . BACKOFF_STEPS . ")"; + warning($warnstr); + dj_sleep($retry_in_sec); + $delay_in_sec = $delay_in_sec * BACKOFF_FACTOR; } - } else { - $old_uuid_pid = 'n/a'; - $needs_cleanup = true; } - - // Either the file didn't exist or we deleted it above. - if (!file_exists($success_file)) { - $oldworkdir = $workdir . '-old-' . getmypid() . '-' . date('Y-m-d_H:i'); - if (!rename($workdir, $oldworkdir)) { - error("Could not rename stale working directory to '$oldworkdir'."); + if (!$succeeded) { + if ($failonerror) { + error($errstr); + } else { + warning($errstr); + $this->endpoints[$this->endpointID]['errorred'] = true; + return null; } - @chmod($oldworkdir, 0700); - warning("Found stale working directory; renamed to '$oldworkdir'."); } + + if ($this->endpoints[$this->endpointID]['errorred']) { + $this->endpoints[$this->endpointID]['errorred'] = false; + $this->endpoints[$this->endpointID]['waiting'] = false; + logmsg(LOG_NOTICE, "Reconnected to endpoint $this->endpointID."); + } + + return $response; } - if ($needs_cleanup && $lastWorkdir !== null) { - cleanup_judging($lastWorkdir); - $lastWorkdir = null; + private function djconfig_refresh(): void + { + $res = $this->request('config', 'GET'); + $res = dj_json_decode($res); + $this->domjudge_config = $res; } + private function djconfig_get_value(string $name) + { + if (empty($this->domjudge_config)) { + $this->djconfig_refresh(); + } - if (!run_command_safe(['mkdir', '-p', "$workdir/compile"])) { - error("Could not create '$workdir/compile'"); + if (!array_key_exists($name, $this->domjudge_config)) { + error("Configuration value '$name' not found in config."); + } + return $this->domjudge_config[$name]; + } + + private function rest_encode_file(string $file, $sizelimit = true): string + { + $maxsize = null; + if ($sizelimit === true) { + $maxsize = (int)$this->djconfig_get_value('output_storage_limit'); + } elseif ($sizelimit === false || $sizelimit == -1) { + $maxsize = -1; + } elseif (is_int($sizelimit) && $sizelimit > 0) { + $maxsize = $sizelimit; + } else { + error("Invalid argument sizelimit = '$sizelimit' specified."); + } + return base64_encode(dj_file_get_contents($file, $maxsize)); + } + + private function usage(): never + { + echo "Usage: " . self::SCRIPT_ID . " [OPTION]...\n" . + "Start the judgedaemon.\n\n" . + " -n bind to CPU and user " . RUNUSER . "-\n" . + " --diskspace-error send internal error on low diskspace; if not set,\n" . + " the judgedaemon will try to clean up and continue\n" . + " -v set verbosity to ; these are syslog levels:\n" . + " default is LOG_INFO = 6, max is LOG_DEBUG = 7\n" . + " -h display this help and exit\n" . + " -V output version information and exit\n\n"; + exit; } - chmod($workdir, 0755); + private function version(): never + { + echo self::SCRIPT_ID . " for DOMjudge version " . DOMJUDGE_VERSION . "\n"; + echo "Written by the DOMjudge developers\n\n"; + echo "DOMjudge comes with ABSOLUTELY NO WARRANTY. This is free software, and you\n"; + echo "are welcome to redistribute it under certain conditions. See the GNU\n"; + echo "General Public Licence for details.\n"; + exit; + } - if (!chdir($workdir)) { - error("Could not chdir to '$workdir'"); + private function read_judgehostlog(int $numLines = 20): string + { + ob_start(); + passthru("tail -n $numLines " . dj_escapeshellarg(LOGFILE)); + return trim(ob_get_clean()); } - if ($lastWorkdir !== $workdir) { - // create chroot environment - logmsg(LOG_INFO, " 🔒 Executing chroot script: '".CHROOT_SCRIPT." start'"); - if (!run_command_safe([LIBJUDGEDIR.'/'.CHROOT_SCRIPT, 'start'], $retval)) { - logmsg(LOG_ERR, "chroot script exited with exitcode $retval"); - disable('judgehost', 'hostname', $myhost, "chroot script exited with exitcode $retval on $myhost"); - continue; + private function run_command_safe(array $command_parts, & $retval = DONT_CARE, $log_nonzero_exitcode = true): bool + { + if (empty($command_parts)) { + logmsg(LOG_WARNING, "Need at least the command that should be called."); + $retval = -1; + return false; } - // Refresh config at start of each batch. - djconfig_refresh(); + $command = implode(' ', array_map('dj_escapeshellarg', $command_parts)); - $lastWorkdir = $workdir; - } + logmsg(LOG_DEBUG, "Executing command: $command"); + system($command, $retval_local); + if ($retval !== DONT_CARE) $retval = $retval_local; - // Make sure the workdir is accessible for the domjudge-run user. - // Will be revoked again after this run finished. - foreach ($row as $judgetask) { - if (!judge($judgetask)) { - // Potentially return remaining outstanding judgetasks here. - $returnedJudgings = request('judgehosts', 'POST', 'hostname=' . urlencode($myhost), false); - if ($returnedJudgings !== null) { - $returnedJudgings = dj_json_decode($returnedJudgings); - foreach ($returnedJudgings as $jud) { - $workdir = judging_directory($workdirpath, $jud); - @chmod($workdir, 0700); - logmsg(LOG_WARNING, " 🔙 Returned unfinished judging with jobid " . $jud['jobid'] . - " in my name; given back unfinished runs from me."); - } + if ($retval_local !== 0) { + if ($log_nonzero_exitcode) { + logmsg(LOG_WARNING, "Command failed with exit code $retval_local: $command"); } - break; + return false; } + + return true; } - file_put_contents($success_file, $expected_uuid_pid); + private function fetch_executable( + string $workdirpath, + string $type, + string $execid, + string $hash, + int $judgeTaskId, + bool $combined_run_compare = false + ): array + { + [$execrunpath, $error, $buildlogpath] = $this->fetch_executable_internal($workdirpath, $type, $execid, $hash, $combined_run_compare); + if (isset($error)) { + $extra_log = null; + if ($buildlogpath !== null) { + $extra_log = dj_file_get_contents($buildlogpath, 4096); + } + logmsg(LOG_ERR, + "Fetching executable failed for $type script '$execid': " . $error); + $description = "$execid: fetch, compile, or deploy of $type script failed."; + $this->disable( + $type . '_script', + $type . '_script_id', + $execid, + $description, + $judgeTaskId, + $extra_log + ); + } + return [$execrunpath, $error]; + } + + private function fetch_executable_internal( + string $workdirpath, + string $type, + string $execid, + string $hash, + bool $combined_run_compare = false + ): array + { + $execdir = join('/', [ + $workdirpath, + 'executable', + $type, + $execid, + $hash + ]); + $execdeploypath = $execdir . '/.deployed'; + $execbuilddir = $execdir . '/build'; + $execbuildpath = $execbuilddir . '/build'; + $execrunpath = $execbuilddir . '/run'; + $execrunjurypath = $execbuilddir . '/runjury'; + if (!is_dir($execdir) || !file_exists($execdeploypath) || + ($combined_run_compare && file_get_contents(LIBJUDGEDIR . '/run-interactive.sh') !== file_get_contents($execrunpath))) { + if (!$this->run_command_safe(['rm', '-rf', $execdir, $execbuilddir])) { + $this->disable('judgehost', 'hostname', $this->myhost, "Deleting '$execdir' or '$execbuilddir' was unsuccessful."); + } + if (!$this->run_command_safe(['mkdir', '-p', $execbuilddir])) { + $this->disable('judgehost', 'hostname', $this->myhost, "Could not create directory '$execbuilddir'"); + } - // Check if we were interrupted while judging, if so, exit (to avoid sleeping) - if ($exitsignalled) { - logmsg(LOG_NOTICE, "Received signal, exiting."); - close_curl_handles(); - fclose($lockfile); - exit; - } + logmsg(LOG_INFO, " 💾 Fetching new executable '$type/$execid' with hash '$hash'."); + $content = $this->request(sprintf('judgehosts/get_files/%s/%s', $type, $execid), 'GET'); + $files = dj_json_decode($content); + unset($content); + $filesArray = []; + foreach ($files as $file) { + $filename = $execbuilddir . '/' . $file['filename']; + $content = base64_decode($file['content']); + file_put_contents($filename, $content); + if ($file['is_executable']) { + chmod($filename, 0755); + } + $filesArray[] = [ + 'hash' => md5($content), + 'filename' => $file['filename'], + 'is_executable' => $file['is_executable'], + ]; + } + unset($files); + uasort($filesArray, fn(array $a, array $b) => strcmp($a['filename'], $b['filename'])); + $computedHash = md5( + join( + array_map( + fn($file) => $file['hash'] . $file['filename'] . $file['is_executable'], + $filesArray + ) + ) + ); + if ($hash !== $computedHash) { + return [null, "Unexpected hash ($computedHash), expected hash: $hash", null]; + } - // restart the judging loop -} + $do_compile = true; + if (!file_exists($execbuildpath)) { + if (file_exists($execrunpath)) { + // 'run' already exists, 'build' does not => don't compile anything + logmsg(LOG_DEBUG, "'run' exists without 'build', we are done."); + $do_compile = false; + } else { + // detect lang and write build file + $buildscript = "#!/bin/sh\n\n"; + $execlang = false; + $source = ""; + $unescapedSource = ""; + foreach ($this->langexts as $lang => $langext) { + if (($handle = opendir($execbuilddir)) === false) { + $this->disable('judgehost', 'hostname', $this->myhost, "Could not open $execbuilddir"); + } + while (($file = readdir($handle)) !== false) { + $ext = pathinfo($file, PATHINFO_EXTENSION); + if (in_array($ext, $langext)) { + $execlang = $lang; + $unescapedSource = $file; + $source = dj_escapeshellarg($unescapedSource); + break; + } + } + closedir($handle); + if ($execlang !== false) { + break; + } + } + if ($execlang === false) { + return [null, "executable must either provide an executable file named 'build' or a C/C++/Java or Python file.", null]; + } + switch ($execlang) { + case 'c': + $buildscript .= "gcc -Wall -O2 -std=gnu11 $source -o run -lm\n"; + break; + case 'cpp': + $buildscript .= "g++ -Wall -O2 -std=gnu++20 $source -o run\n"; + break; + case 'java': + $buildscript .= "javac -cp . -d . $source\n"; + $buildscript .= "echo '#!/bin/sh' > run\n"; + // no main class detection here + $buildscript .= "echo 'COMPARE_DIR=\$(dirname \"\$0\")' >> run\n"; + $mainClass = basename($unescapedSource, '.java'); + // Note: since the $@ is within single quotes, we do not need to double escape it. + $buildscript .= "echo 'java -cp \"\$COMPARE_DIR\" $mainClass \"\$@\"' >> run\n"; + $buildscript .= "chmod +x run\n"; + break; + case 'py': + $buildscript .= "echo '#!/bin/sh' > run\n"; + $buildscript .= "echo 'COMPARE_DIR=\$(dirname \"\$0\")' >> run\n"; + // Note: since the $@ is within single quotes, we do not need to double escape it. + $buildscript .= "echo 'python3 \"\$COMPARE_DIR/$source\" \"\$@\"' >> run\n"; + $buildscript .= "chmod +x run\n"; + break; + } + if (file_put_contents($execbuildpath, $buildscript) === false) { + $this->disable('judgehost', 'hostname', $this->myhost, "Could not write file 'build' in $execbuilddir"); + } + chmod($execbuildpath, 0755); + } + } elseif (!is_executable($execbuildpath)) { + return [null, "Invalid executable, file 'build' exists but is not executable.", null]; + } -function registerJudgehost(string $myhost): void -{ - global $endpoints, $endpointID; - $endpoint = &$endpoints[$endpointID]; - - // Only try to register every 30s. - $now = time(); - if ($now - $endpoint['last_attempt'] < 30) { - $endpoint['waiting'] = true; - return; - } - $endpoint['last_attempt'] = $now; + if ($do_compile) { + logmsg(LOG_DEBUG, "Building executable in $execdir, under 'build/'"); - logmsg(LOG_NOTICE, "Registering judgehost on endpoint $endpointID: " . $endpoint['url']); - $endpoints[$endpointID]['ch'] = setup_curl_handle($endpoint['user'], $endpoint['pass']); + putenv('SCRIPTTIMELIMIT=' . $this->djconfig_get_value('script_timelimit')); + putenv('SCRIPTMEMLIMIT=' . $this->djconfig_get_value('script_memory_limit')); + putenv('SCRIPTFILELIMIT=' . $this->djconfig_get_value('script_filesize_limit')); - // Create directory where to test submissions - $workdirpath = JUDGEDIR . "/$myhost/endpoint-$endpointID"; - if (!run_command_safe(['mkdir', '-p', "$workdirpath/testcase"])) { - error("Could not create $workdirpath"); - } - chmod("$workdirpath/testcase", 0700); - - // Auto-register judgehost. - // If there are any unfinished judgings in the queue in my name, - // they have and will not be finished. Give them back. - $unfinished = request('judgehosts', 'POST', 'hostname=' . urlencode($myhost), false); - if ($unfinished === null) { - logmsg(LOG_WARNING, "Registering judgehost on endpoint $endpointID failed."); - } else { - $unfinished = dj_json_decode($unfinished); - foreach ($unfinished as $jud) { - $workdir = judging_directory($workdirpath, $jud); - @chmod($workdir, 0700); - logmsg(LOG_WARNING, "Found unfinished judging with jobid " . $jud['jobid'] . - " in my name; given back unfinished runs from me."); + if (!$this->run_command_safe([LIBJUDGEDIR . '/build_executable.sh', $execdir])) { + return [null, "Failed to build executable in $execdir.", "$execdir/build.log"]; + } + chmod($execrunpath, 0755); + } + if (!is_file($execrunpath) || !is_executable($execrunpath)) { + return [null, "Invalid build file, must produce an executable file 'run'.", null]; + } + if ($combined_run_compare) { + # For combined run and compare (i.e. for interactive problems), we + # need to wrap the jury provided 'run' script with 'runpipe' to + # handle the bidirectional communication. First 'run' is renamed to + # 'runjury', and then replaced by the script below, which runs the + # team submission and runjury programs and connects their pipes. + $runscript = file_get_contents(LIBJUDGEDIR . '/run-interactive.sh'); + if (rename($execrunpath, $execrunjurypath) === false) { + $this->disable('judgehost', 'hostname', $this->myhost, "Could not move file 'run' to 'runjury' in $execbuilddir"); + } + if (file_put_contents($execrunpath, $runscript) === false) { + $this->disable('judgehost', 'hostname', $this->myhost, "Could not write file 'run' in $execbuilddir"); + } + chmod($execrunpath, 0755); + } + + if (!is_file($execrunpath) || !is_executable($execrunpath)) { + return [null, "Invalid build file, must produce an executable file 'run'.", null]; + } + + // Create file to mark executable successfully deployed. + touch($execdeploypath); } - } -} -function disable( - string $kind, - string $idcolumn, - $id, - string $description, - ?int $judgeTaskId = null, - ?string $extra_log = null -): void { - global $myhost; - $disabled = dj_json_encode(['kind' => $kind, $idcolumn => $id]); - $judgehostlog = read_judgehostlog(); - if (isset($extra_log)) { - $judgehostlog .= "\n\n" - . "--------------------------------------------------------------------------------" - . "\n\n" - . $extra_log; - } - $args = 'description=' . urlencode($description) . - '&judgehostlog=' . urlencode(base64_encode($judgehostlog)) . - '&disabled=' . urlencode($disabled) . - '&hostname=' . urlencode($myhost); - if (isset($judgeTaskId)) { - $args .= '&judgetaskid=' . urlencode((string)$judgeTaskId); + return [$execrunpath, null, null]; } - $error_id = request('judgehosts/internal-error', 'POST', $args); - logmsg(LOG_ERR, "=> internal error " . $error_id); -} + private function registerJudgehost(): void + { + $endpoint = &$this->endpoints[$this->endpointID]; -function read_metadata(string $filename): ?array -{ - if (!is_readable($filename)) { - return null; - } + // Only try to register every 30s. + $now = time(); + if ($now - $endpoint['last_attempt'] < 30) { + $endpoint['waiting'] = true; + return; + } + $endpoint['last_attempt'] = $now; + + logmsg(LOG_NOTICE, "Registering judgehost on endpoint $this->endpointID: " . $endpoint['url']); + $this->endpoints[$this->endpointID]['ch'] = $this->setup_curl_handle($endpoint['user'], $endpoint['pass']); - // Don't quite treat it as YAML, but simply key/value pairs. - $contents = explode("\n", dj_file_get_contents($filename)); - $res = []; - foreach ($contents as $line) { - if (str_contains($line, ":")) { - [$key, $value] = explode(":", $line, 2); - $res[$key] = trim($value); + // Create directory where to test submissions + $workdirpath = JUDGEDIR . "/$this->myhost/endpoint-$this->endpointID"; + if (!$this->run_command_safe(['mkdir', '-p', "$workdirpath/testcase"])) { + error("Could not create $workdirpath"); + } + chmod("$workdirpath/testcase", 0700); + + // Auto-register judgehost. + // If there are any unfinished judgings in the queue in my name, + // they have and will not be finished. Give them back. + $unfinished = $this->request('judgehosts', 'POST', ['hostname' => urlencode($this->myhost)], false); + if ($unfinished === null) { + logmsg(LOG_WARNING, "Registering judgehost on endpoint $this->endpointID failed."); + } else { + $unfinished = dj_json_decode($unfinished); + foreach ($unfinished as $jud) { + $workdir = $this->judging_directory($workdirpath, $jud); + @chmod($workdir, 0700); + logmsg(LOG_WARNING, "Found unfinished judging with jobid " . $jud['jobid'] . + " in my name; given back unfinished runs from me."); + } } } - return $res; -} + private function disable( + string $kind, + string $idcolumn, + $id, + string $description, + ?int $judgeTaskId = null, + ?string $extra_log = null + ): void + { + $disabled = dj_json_encode(['kind' => $kind, $idcolumn => $id]); + $judgehostlog = $this->read_judgehostlog(); + if (isset($extra_log)) { + $judgehostlog .= "\n\n" + . "--------------------------------------------------------------------------------" + . "\n\n" + . $extra_log; + } + $args = 'description=' . urlencode($description) . + '&judgehostlog=' . urlencode(base64_encode($judgehostlog)) . + '&disabled=' . urlencode($disabled) . + '&hostname=' . urlencode($this->myhost); + if (isset($judgeTaskId)) { + $args .= '&judgetaskid=' . urlencode((string)$judgeTaskId); + } -function cleanup_judging(string $workdir) : void -{ - global $myhost; - // revoke readablity for domjudge-run user to this workdir - chmod($workdir, 0700); - - // destroy chroot environment - logmsg(LOG_INFO, " 🔓 Executing chroot script: '".CHROOT_SCRIPT." stop'"); - if (!run_command_safe([LIBJUDGEDIR.'/'.CHROOT_SCRIPT, 'stop'], $retval)) { - logmsg(LOG_ERR, "chroot script exited with exitcode $retval"); - disable('judgehost', 'hostname', $myhost, "chroot script exited with exitcode $retval on $myhost"); - // Just continue here: even though we might continue a current - // compile/test-run cycle, we don't know whether we're in one here, - // and worst case, the chroot script will fail the next time when - // starting. + $error_id = $this->request('judgehosts/internal-error', 'POST', $args); + logmsg(LOG_ERR, "=> internal error " . $error_id); } - // Evict all contents of the workdir from the kernel fs cache - if (!run_command_safe([LIBJUDGEDIR . '/evict', $workdir])) { - warning("evict script failed, continuing gracefully"); + private function read_metadata(string $filename): ?array + { + if (!is_readable($filename)) { + return null; + } + + // Don't quite treat it as YAML, but simply key/value pairs. + $contents = explode("\n", dj_file_get_contents($filename)); + $res = []; + foreach ($contents as $line) { + if (str_contains($line, ":")) { + [$key, $value] = explode(":", $line, 2); + $res[$key] = trim($value); + } + } + + return $res; } -} -function compile( - array $judgeTask, - string $workdir, - string $workdirpath, - array $compile_config, - ?string $daemonid, - int $output_storage_limit -): bool { - global $myhost, $EXITCODES; - - // Reuse compilation if it already exists. - if (file_exists("$workdir/compile.success")) { - return true; + private function cleanup_judging(string $workdir): void + { + // revoke readablity for domjudge-run user to this workdir + chmod($workdir, 0700); + + // destroy chroot environment + logmsg(LOG_INFO, " 🔓 Executing chroot script: '" . self::CHROOT_SCRIPT . " stop'"); + if (!$this->run_command_safe([LIBJUDGEDIR . '/' . self::CHROOT_SCRIPT, 'stop'], $retval)) { + logmsg(LOG_ERR, "chroot script exited with exitcode $retval"); + $this->disable('judgehost', 'hostname', $this->myhost, "chroot script exited with exitcode $retval on $this->myhost"); + // Just continue here: even though we might continue a current + // compile/test-run cycle, we don't know whether we're in one here, + // and worst case, the chroot script will fail the next time when + // starting. + } + + // Evict all contents of the workdir from the kernel fs cache + if (!$this->run_command_safe([LIBJUDGEDIR . '/evict', $workdir])) { + warning("evict script failed, continuing gracefully"); + } } - // Verify compile and runner versions. - $judgeTaskId = $judgeTask['judgetaskid']; - $version_verification = dj_json_decode(request('judgehosts/get_version_commands/' . $judgeTaskId, 'GET')); - if (isset($version_verification['compiler_version_command']) || isset($version_verification['runner_version_command'])) { - logmsg(LOG_INFO, " 📋 Verifying versions."); - $versions = []; - $version_output_file = $workdir . '/version_check.out'; - $args = 'hostname=' . urlencode($myhost); - foreach (['compiler', 'runner'] as $type) { - if (isset($version_verification[$type . '_version_command'])) { - if (file_exists($version_output_file)) { - unlink($version_output_file); - } + private function compile( + array $judgeTask, + string $workdir, + string $workdirpath, + array $compile_config, + ?string $daemonid, + int $output_storage_limit + ): bool + { + // Reuse compilation if it already exists. + if (file_exists("$workdir/compile.success")) { + return true; + } - $vcscript_content = $version_verification[$type . '_version_command']; - $vcscript = tempnam(TMPDIR, 'version_check-'); - file_put_contents($vcscript, $vcscript_content); - chmod($vcscript, 0755); + // Verify compile and runner versions. + $judgeTaskId = $judgeTask['judgetaskid']; + $version_verification = dj_json_decode($this->request('judgehosts/get_version_commands/' . $judgeTaskId, 'GET')); + if (isset($version_verification['compiler_version_command']) || isset($version_verification['runner_version_command'])) { + logmsg(LOG_INFO, " 📋 Verifying versions."); + $versions = []; + $version_output_file = $workdir . '/version_check.out'; + $args = 'hostname=' . urlencode($this->myhost); + foreach (['compiler', 'runner'] as $type) { + if (isset($version_verification[$type . '_version_command'])) { + if (file_exists($version_output_file)) { + unlink($version_output_file); + } - run_command_safe([LIBJUDGEDIR . "/version_check.sh", $vcscript, $workdir], $retval); + $vcscript_content = $version_verification[$type . '_version_command']; + $vcscript = tempnam(TMPDIR, 'version_check-'); + file_put_contents($vcscript, $vcscript_content); + chmod($vcscript, 0755); - $versions[$type] = trim(file_get_contents($version_output_file)); - if ($retval !== 0) { - $versions[$type] = - "Getting $type version failed with exit code $retval\n" - . $versions[$type]; - } + $this->run_command_safe([LIBJUDGEDIR . "/version_check.sh", $vcscript, $workdir], $retval); - unlink($vcscript); - } - if (isset($versions[$type])) { - $args .= "&$type=" . urlencode(base64_encode($versions[$type])); + $versions[$type] = trim(file_get_contents($version_output_file)); + if ($retval !== 0) { + $versions[$type] = + "Getting $type version failed with exit code $retval\n" + . $versions[$type]; + } + + unlink($vcscript); + } + if (isset($versions[$type])) { + $args .= "&$type=" . urlencode(base64_encode($versions[$type])); + } } - } - // TODO: Add actual check once implemented in the backend. - request('judgehosts/check_versions/' . $judgeTaskId, 'PUT', $args); - } + // TODO: Add actual check once implemented in the backend. + $this->request('judgehosts/check_versions/' . $judgeTaskId, 'PUT', $args); + } - // Get the source code from the DB and store in local file(s). - $url = sprintf('judgehosts/get_files/source/%s', $judgeTask['submitid']); - $sources = request($url, 'GET'); - $sources = dj_json_decode($sources); - $files = []; - $hasFiltered = false; - foreach ($sources as $source) { - $srcfile = "$workdir/compile/$source[filename]"; - $file = $source['filename']; - if ($compile_config['filter_compiler_files']) { - $picked = false; - foreach ($compile_config['language_extensions'] as $extension) { - $extensionLength = strlen($extension); - if (substr($file, -$extensionLength) === $extension) { - $files[] = $file; - $picked = true; - break; + // Get the source code from the DB and store in local file(s). + $url = sprintf('judgehosts/get_files/source/%s', $judgeTask['submitid']); + $sources = $this->request($url, 'GET'); + $sources = dj_json_decode($sources); + $files = []; + $hasFiltered = false; + foreach ($sources as $source) { + $srcfile = "$workdir/compile/$source[filename]"; + $file = $source['filename']; + if ($compile_config['filter_compiler_files']) { + $picked = false; + foreach ($compile_config['language_extensions'] as $extension) { + $extensionLength = strlen($extension); + if (substr($file, -$extensionLength) === $extension) { + $files[] = $file; + $picked = true; + break; + } + } + if (!$picked) { + $hasFiltered = true; } + } else { + $files[] = $file; } - if (!$picked) { - $hasFiltered = true; + if (file_put_contents($srcfile, base64_decode($source['content'])) === false) { + error("Could not create $srcfile"); } - } else { - $files[] = $file; } - if (file_put_contents($srcfile, base64_decode($source['content'])) === false) { - error("Could not create $srcfile"); + + if (empty($files) && $hasFiltered) { + // Note: It may be tempting to assume that this codepath can be never + // reached since we prevent these submissions from being submitted both + // via command line and the web interface. However, the code path can + // be triggered when the filtering is activated between submission and + // rejudge. + $message = 'No files with allowed extensions found to pass to compiler. Allowed extensions: ' + . implode(', ', $compile_config['language_extensions']); + $args = 'compile_success=0' . + '&output_compile=' . urlencode(base64_encode($message)); + + $url = sprintf('judgehosts/update-judging/%s/%s', urlencode($this->myhost), urlencode((string)$judgeTask['judgetaskid'])); + $this->request($url, 'PUT', $args); + + // Revoke readablity for domjudge-run user to this workdir. + chmod($workdir, 0700); + logmsg(LOG_NOTICE, "Judging s$judgeTask[submitid], task $judgeTask[judgetaskid]: compile error"); + return false; } - } - if (empty($files) && $hasFiltered) { - // Note: It may be tempting to assume that this codepath can be never - // reached since we prevent these submissions from being submitted both - // via command line and the web interface. However, the code path can - // be triggered when the filtering is activated between submission and - // rejudge. - $message = 'No files with allowed extensions found to pass to compiler. Allowed extensions: ' - . implode(', ', $compile_config['language_extensions']); - $args = 'compile_success=0' . - '&output_compile=' . urlencode(base64_encode($message)); - - $url = sprintf('judgehosts/update-judging/%s/%s', urlencode($myhost), urlencode((string)$judgeTask['judgetaskid'])); - request($url, 'PUT', $args); - - // Revoke readablity for domjudge-run user to this workdir. - chmod($workdir, 0700); - logmsg(LOG_NOTICE, "Judging s$judgeTask[submitid], task $judgeTask[judgetaskid]: compile error"); - return false; - } + if (count($files) == 0) { + error("No submission files could be downloaded."); + } - if (count($files)==0) { - error("No submission files could be downloaded."); - } + [$execrunpath, $error] = $this->fetch_executable( + $workdirpath, + 'compile', + $judgeTask['compile_script_id'], + $compile_config['hash'], + $judgeTask['judgetaskid'] + ); + if (isset($error)) { + return false; + } - [$execrunpath, $error] = fetch_executable( - $workdirpath, - 'compile', - $judgeTask['compile_script_id'], - $compile_config['hash'], - $judgeTask['judgetaskid'] - ); - if (isset($error)) { - return false; - } + // Compile the program. + $compile_command_parts = [LIBJUDGEDIR . '/compile.sh']; + if (isset($daemonid)) { + $compile_command_parts[] = '-n'; + $compile_command_parts[] = $daemonid; + } + array_push($compile_command_parts, $execrunpath, $workdir, ...$files); + // Note that the $retval is handled further down after reading/writing metadata. + $this->run_command_safe($compile_command_parts, $retval, log_nonzero_exitcode: false); - // Compile the program. - $compile_command_parts = [LIBJUDGEDIR . '/compile.sh']; - if (isset($daemonid)) { - $compile_command_parts[] = '-n'; - $compile_command_parts[] = $daemonid; - } - array_push($compile_command_parts, $execrunpath, $workdir, ...$files); - // Note that the $retval is handled further down after reading/writing metadata. - run_command_safe($compile_command_parts, $retval, log_nonzero_exitcode: false); + $compile_output = ''; + if (is_readable($workdir . '/compile.out')) { + $compile_output = dj_file_get_contents($workdir . '/compile.out', 50000); + } + if (empty($compile_output) && is_readable($workdir . '/compile.tmp')) { + $compile_output = dj_file_get_contents($workdir . '/compile.tmp', 50000); + } - $compile_output = ''; - if (is_readable($workdir . '/compile.out')) { - $compile_output = dj_file_get_contents($workdir . '/compile.out', 50000); - } - if (empty($compile_output) && is_readable($workdir . '/compile.tmp')) { - $compile_output = dj_file_get_contents($workdir . '/compile.tmp', 50000); - } + // Try to read metadata from file + $metadata = $this->read_metadata($workdir . '/compile.meta'); + if (isset($metadata['internal-error'])) { + alert('error'); + $internalError = $metadata['internal-error']; + $compile_output .= "\n--------------------------------------------------------------------------------\n\n" . + "Internal errors reported:\n" . $internalError; + + if (str_starts_with($internalError, 'compile script: ')) { + $internalError = preg_replace('/^compile script: /', '', $internalError); + $description = "The compile script returned an error: $internalError"; + $this->disable('compile_script', 'compile_script_id', $judgeTask['compile_script_id'], $description, $judgeTask['judgetaskid'], $compile_output); + } else { + $description = "Running compile.sh caused an error/crash: $internalError"; + // Note we are disabling the judgehost in this case since it's + // likely an error intrinsic to this judgehost's setup, e.g. + // missing cgroups. + $this->disable('judgehost', 'hostname', $this->myhost, $description, $judgeTask['judgetaskid'], $compile_output); + } + logmsg(LOG_ERR, $description); - // Try to read metadata from file - $metadata = read_metadata($workdir . '/compile.meta'); - if (isset($metadata['internal-error'])) { - alert('error'); - $internalError = $metadata['internal-error']; - $compile_output .= "\n--------------------------------------------------------------------------------\n\n". - "Internal errors reported:\n".$internalError; - - if (str_starts_with($internalError, 'compile script: ')) { - $internalError = preg_replace('/^compile script: /', '', $internalError); - $description = "The compile script returned an error: $internalError"; - disable('compile_script', 'compile_script_id', $judgeTask['compile_script_id'], $description, $judgeTask['judgetaskid'], $compile_output); - } else { - $description = "Running compile.sh caused an error/crash: $internalError"; - // Note we are disabling the judgehost in this case since it's - // likely an error intrinsic to this judgehost's setup, e.g. - // missing cgroups. - disable('judgehost', 'hostname', $myhost, $description, $judgeTask['judgetaskid'], $compile_output); + return false; } - logmsg(LOG_ERR, $description); - return false; - } + // What does the exitcode mean? + if (!isset($this->EXITCODES[$retval])) { + alert('error'); + $description = "Unknown exitcode from compile.sh for s$judgeTask[submitid]: $retval"; + logmsg(LOG_ERR, $description); + $this->disable('compile_script', 'compile_script_id', $judgeTask['compile_script_id'], $description, $judgeTask['judgetaskid'], $compile_output); - // What does the exitcode mean? - if (! isset($EXITCODES[$retval])) { - alert('error'); - $description = "Unknown exitcode from compile.sh for s$judgeTask[submitid]: $retval"; - logmsg(LOG_ERR, $description); - disable('compile_script', 'compile_script_id', $judgeTask['compile_script_id'], $description, $judgeTask['judgetaskid'], $compile_output); + return false; + } - return false; - } + logmsg(LOG_INFO, " 💻 Compilation: ($files[0]) '" . $this->EXITCODES[$retval] . "'"); + $compile_success = ($this->EXITCODES[$retval] === 'correct'); - logmsg(LOG_INFO, " 💻 Compilation: ($files[0]) '".$EXITCODES[$retval]."'"); - $compile_success = ($EXITCODES[$retval]==='correct'); + // Pop the compilation result back into the judging table. + $args = 'compile_success=' . $compile_success . + '&output_compile=' . urlencode($this->rest_encode_file($workdir . '/compile.out', $output_storage_limit)) . + '&compile_metadata=' . urlencode($this->rest_encode_file($workdir . '/compile.meta', false)); + if (isset($metadata['entry_point'])) { + $args .= '&entry_point=' . urlencode($metadata['entry_point']); + } - // Pop the compilation result back into the judging table. - $args = 'compile_success=' . $compile_success . - '&output_compile=' . urlencode(rest_encode_file($workdir . '/compile.out', $output_storage_limit)) . - '&compile_metadata=' . urlencode(rest_encode_file($workdir . '/compile.meta', false)); - if (isset($metadata['entry_point'])) { - $args .= '&entry_point=' . urlencode($metadata['entry_point']); - } + $url = sprintf('judgehosts/update-judging/%s/%s', urlencode($this->myhost), urlencode((string)$judgeTask['judgetaskid'])); + $this->request($url, 'PUT', $args); + + // Compile error: our job here is done. + if (!$compile_success) { + return false; + } - $url = sprintf('judgehosts/update-judging/%s/%s', urlencode($myhost), urlencode((string)$judgeTask['judgetaskid'])); - request($url, 'PUT', $args); + touch("$workdir/compile.success"); - // Compile error: our job here is done. - if (! $compile_success) { - return false; + return true; } - touch("$workdir/compile.success"); + private function compile_and_run_submission(array $judgeTask, string $workdirpath): bool + { + $startTime = microtime(true); - return true; -} + $compile_config = dj_json_decode($judgeTask['compile_config']); + $run_config = dj_json_decode($judgeTask['run_config']); + $compare_config = dj_json_decode($judgeTask['compare_config']); -function judge(array $judgeTask): bool -{ - global $EXITCODES, $myhost, $options, $workdirpath, $exitsignalled, $gracefulexitsignalled, $endpointID; - $startTime = microtime(true); - - $compile_config = dj_json_decode($judgeTask['compile_config']); - $run_config = dj_json_decode($judgeTask['run_config']); - $compare_config = dj_json_decode($judgeTask['compare_config']); - - // Set configuration variables for called programs - putenv('CREATE_WRITABLE_TEMP_DIR=' . (CREATE_WRITABLE_TEMP_DIR ? '1' : '')); - - // These are set again below before comparing. - putenv('SCRIPTTIMELIMIT=' . $compile_config['script_timelimit']); - putenv('SCRIPTMEMLIMIT=' . $compile_config['script_memory_limit']); - putenv('SCRIPTFILELIMIT=' . $compile_config['script_filesize_limit']); - - putenv('MEMLIMIT=' . $run_config['memory_limit']); - putenv('FILELIMIT=' . $run_config['output_limit']); - putenv('PROCLIMIT=' . $run_config['process_limit']); - if ($run_config['entry_point'] !== null) { - putenv('ENTRY_POINT=' . $run_config['entry_point']); - } else { - putenv('ENTRY_POINT'); - } - $output_storage_limit = (int) djconfig_get_value('output_storage_limit'); + // Set configuration variables for called programs + putenv('CREATE_WRITABLE_TEMP_DIR=' . (CREATE_WRITABLE_TEMP_DIR ? '1' : '')); - $cpuset_opt = ""; - if (isset($options['daemonid'])) { - $cpuset_opt = '-n ' . dj_escapeshellarg($options['daemonid']); - } + // These are set again below before comparing. + putenv('SCRIPTTIMELIMIT=' . $compile_config['script_timelimit']); + putenv('SCRIPTMEMLIMIT=' . $compile_config['script_memory_limit']); + putenv('SCRIPTFILELIMIT=' . $compile_config['script_filesize_limit']); - $workdir = judging_directory($workdirpath, $judgeTask); - $compile_success = compile($judgeTask, $workdir, $workdirpath, $compile_config, $options['daemonid'] ?? null, $output_storage_limit); - if (!$compile_success) { - return false; - } - - // TODO: How do we plan to handle these? - $overshoot = djconfig_get_value('timelimit_overshoot'); + putenv('MEMLIMIT=' . $run_config['memory_limit']); + putenv('FILELIMIT=' . $run_config['output_limit']); + putenv('PROCLIMIT=' . $run_config['process_limit']); + if ($run_config['entry_point'] !== null) { + putenv('ENTRY_POINT=' . $run_config['entry_point']); + } else { + putenv('ENTRY_POINT'); + } + $output_storage_limit = (int)$this->djconfig_get_value('output_storage_limit'); - // Check whether we have received an exit signal (but not a graceful exit signal). - if (function_exists('pcntl_signal_dispatch')) { - pcntl_signal_dispatch(); - } - if ($exitsignalled && !$gracefulexitsignalled) { - logmsg(LOG_NOTICE, "Received HARD exit signal, aborting current judging."); + $cpuset_opt = ""; + if (isset($this->options['daemonid'])) { + $cpuset_opt = '-n ' . dj_escapeshellarg($this->options['daemonid']); + } - // Make sure the domserver knows that we didn't finish this judging. - $unfinished = request('judgehosts', 'POST', 'hostname=' . urlencode($myhost)); - $unfinished = dj_json_decode($unfinished); - foreach ($unfinished as $jud) { - logmsg(LOG_WARNING, "Aborted judging task " . $jud['judgetaskid'] . - " due to signal"); + $workdir = $this->judging_directory($workdirpath, $judgeTask); + $compile_success = $this->compile($judgeTask, $workdir, $workdirpath, $compile_config, $this->options['daemonid'] ?? null, $output_storage_limit); + if (!$compile_success) { + return false; } - return false; - } - logmsg(LOG_INFO, " 🏃 Running testcase $judgeTask[testcase_id]..."); - $testcasedir = $workdir . "/testcase" . sprintf('%05d', $judgeTask['testcase_id']); - $tcfile = fetchTestcase($workdirpath, $judgeTask['testcase_id'], $judgeTask['judgetaskid'], $judgeTask['testcase_hash']); - if ($tcfile === null) { - // error while fetching testcase - return false; - } + // TODO: How do we plan to handle these? + $overshoot = $this->djconfig_get_value('timelimit_overshoot'); - // do the actual test-run - $combined_run_compare = $compare_config['combined_run_compare']; - [$run_runpath, $error] = fetch_executable( - $workdirpath, - 'run', - $judgeTask['run_script_id'], - $run_config['hash'], - $judgeTask['judgetaskid'], - $combined_run_compare); - if (isset($error)) { - return false; - } + // Check whether we have received an exit signal (but not a graceful exit signal). + if (function_exists('pcntl_signal_dispatch')) { + pcntl_signal_dispatch(); + } + if ($this->exitsignalled && !$this->gracefulexitsignalled) { + logmsg(LOG_NOTICE, "Received HARD exit signal, aborting current judging."); + + // Make sure the domserver knows that we didn't finish this judging. + $unfinished = $this->request('judgehosts', 'POST', ['hostname' => urlencode($this->myhost)]); + $unfinished = dj_json_decode($unfinished); + foreach ($unfinished as $jud) { + logmsg(LOG_WARNING, "Aborted judging task " . $jud['judgetaskid'] . + " due to signal"); + } + return false; + } + + return $this->run_testcase($judgeTask, $workdir, $workdirpath, $run_config, $compare_config, $output_storage_limit, $overshoot, $startTime); + } + + private function run_testcase( + array $judgeTask, + string $workdir, + string $workdirpath, + array $run_config, + array $compare_config, + int $output_storage_limit, + string $overshoot, + float $startTime + ): bool { + logmsg(LOG_INFO, " 🏃 Running testcase $judgeTask[testcase_id]..."); + $testcasedir = $workdir . "/testcase" . sprintf('%05d', $judgeTask['testcase_id']); + $tcfile = $this->fetchTestcase($workdirpath, $judgeTask['testcase_id'], $judgeTask['judgetaskid'], $judgeTask['testcase_hash']); + if ($tcfile === null) { + // error while fetching testcase + return false; + } - if ($combined_run_compare) { - // set to empty string to signal the testcase_run script that the - // run script also acts as compare script - $compare_runpath = ''; - } else { - [$compare_runpath, $error] = fetch_executable( + // do the actual test-run + $combined_run_compare = $compare_config['combined_run_compare']; + [$run_runpath, $error] = $this->fetch_executable( $workdirpath, - 'compare', - $judgeTask['compare_script_id'], - $compare_config['hash'], - $judgeTask['judgetaskid'] - ); + 'run', + $judgeTask['run_script_id'], + $run_config['hash'], + $judgeTask['judgetaskid'], + $combined_run_compare); if (isset($error)) { return false; } - } - $hardtimelimit = $run_config['time_limit'] - + overshoot_time($run_config['time_limit'], $overshoot) - + $run_config['overshoot']; - $timelimit = [ - 'cpu' => [ $run_config['time_limit'], $hardtimelimit ], - 'wall' => [ $run_config['time_limit'], $hardtimelimit ], - ]; - if ($combined_run_compare) { - // This accounts for wall time spent in the validator. We may likely - // want to make this configurable in the future. The current factor is - // under the assumption that the validator has to do approximately the - // same amount of work wall-time wise as the submission. - $timelimit['wall'][1] *= 2; - } - - // While we already set those above to likely the same values from the - // compile config, we do set them again from the compare config here. - putenv('SCRIPTTIMELIMIT=' . $compare_config['script_timelimit']); - putenv('SCRIPTMEMLIMIT=' . $compare_config['script_memory_limit']); - putenv('SCRIPTFILELIMIT=' . $compare_config['script_filesize_limit']); - - $input = $tcfile['input']; - $output = $tcfile['output']; - $passLimit = $run_config['pass_limit'] ?? 1; - for ($passCnt = 1; $passCnt <= $passLimit; $passCnt++) { - $nextPass = false; - if ($passLimit > 1) { - logmsg(LOG_INFO, " 🔄 Running pass $passCnt..."); - } - - $passdir = $testcasedir . '/' . $passCnt; - mkdir($passdir, 0755, true); - - // In multi-pass problems, all files in the feedback directory - // are guaranteed to persist between passes, except `nextpass.in`. - // So, we recursively copy the feedback directory for every pass - // after the first (note that $passCnt starts at 1). - if ($passCnt > 1) { - $prevPassdir = $testcasedir . '/' . ($passCnt - 1) . '/feedback'; - run_command_safe(['cp', '-R', $prevPassdir, $passdir . '/']); - run_command_safe(['rm', $passdir . '/feedback/nextpass.in']); - } - - // Copy program with all possible additional files to testcase - // dir. Use hardlinks to preserve space with big executables. - $programdir = $passdir . '/execdir'; - if (!run_command_safe(['mkdir', '-p', $programdir])) { - error("Could not create directory '$programdir'"); - } - - foreach (glob("$workdir/compile/*") as $compile_file) { - if (!run_command_safe(['cp', '-PRl', $compile_file, $programdir])) { - error("Could not copy program to '$programdir'"); - } - } - - $timelimit_str = implode(':', $timelimit['cpu']) . ',' . implode(':', $timelimit['wall']); - $run_command_parts = [LIBJUDGEDIR . '/testcase_run.sh']; - if (isset($options['daemonid'])) { - $run_command_parts[] = '-n'; - $run_command_parts[] = $options['daemonid']; - } - array_push($run_command_parts, - $input, - $output, - $timelimit_str, - $passdir, - $run_runpath, - $compare_runpath, - $compare_config['compare_args'] - ); - run_command_safe($run_command_parts, $retval, log_nonzero_exitcode: false); + if ($combined_run_compare) { + // set to empty string to signal the testcase_run script that the + // run script also acts as compare script + $compare_runpath = ''; + } else { + [$compare_runpath, $error] = $this->fetch_executable( + $workdirpath, + 'compare', + $judgeTask['compare_script_id'], + $compare_config['hash'], + $judgeTask['judgetaskid'] + ); + if (isset($error)) { + return false; + } + } - // What does the exitcode mean? - if (!isset($EXITCODES[$retval])) { - alert('error'); - error("Unknown exitcode ($retval) from testcase_run.sh for s$judgeTask[submitid]"); + $hardtimelimit = $run_config['time_limit'] + + overshoot_time($run_config['time_limit'], $overshoot) + + $run_config['overshoot']; + $timelimit = [ + 'cpu' => [$run_config['time_limit'], $hardtimelimit], + 'wall' => [$run_config['time_limit'], $hardtimelimit], + ]; + if ($combined_run_compare) { + // This accounts for wall time spent in the validator. We may likely + // want to make this configurable in the future. The current factor is + // under the assumption that the validator has to do approximately the + // same amount of work wall-time wise as the submission. + $timelimit['wall'][1] *= 2; } - $result = $EXITCODES[$retval]; - // Try to read metadata from file - $runtime = null; - $metadata = read_metadata($passdir . '/program.meta'); + // While we already set those above to likely the same values from the + // compile config, we do set them again from the compare config here. + putenv('SCRIPTTIMELIMIT=' . $compare_config['script_timelimit']); + putenv('SCRIPTMEMLIMIT=' . $compare_config['script_memory_limit']); + putenv('SCRIPTFILELIMIT=' . $compare_config['script_filesize_limit']); + + $input = $tcfile['input']; + $output = $tcfile['output']; + $passLimit = $run_config['pass_limit'] ?? 1; + for ($passCnt = 1; $passCnt <= $passLimit; $passCnt++) { + $nextPass = false; + if ($passLimit > 1) { + logmsg(LOG_INFO, " 🔄 Running pass $passCnt..."); + } - if (isset($metadata['time-used']) && array_key_exists($metadata['time-used'], $metadata)) { - $runtime = $metadata[$metadata['time-used']]; - } + $passdir = $testcasedir . '/' . $passCnt; + mkdir($passdir, 0755, true); + + // In multi-pass problems, all files in the feedback directory + // are guaranteed to persist between passes, except `nextpass.in`. + // So, we recursively copy the feedback directory for every pass + // after the first (note that $passCnt starts at 1). + if ($passCnt > 1) { + $prevPassdir = $testcasedir . '/' . ($passCnt - 1) . '/feedback'; + $this->run_command_safe(['cp', '-R', $prevPassdir, $passdir . '/']); + $this->run_command_safe(['rm', $passdir . '/feedback/nextpass.in']); + } - if ($result === 'compare-error') { - $compareMeta = read_metadata($passdir . '/compare.meta'); - $compareExitCode = 'n/a'; - if (isset($compareMeta['exitcode'])) { - $compareExitCode = $compareMeta['exitcode']; + // Copy program with all possible additional files to testcase + // dir. Use hardlinks to preserve space with big executables. + $programdir = $passdir . '/execdir'; + if (!$this->run_command_safe(['mkdir', '-p', $programdir])) { + error("Could not create directory '$programdir'"); } - if ($combined_run_compare) { - logmsg(LOG_ERR, "comparing failed for combined run/compare script '" . $judgeTask['run_script_id'] . "'"); - $description = 'combined run/compare script ' . $judgeTask['run_script_id'] . ' crashed with exit code ' . $compareExitCode . ", expected one of 42/43"; - disable('run_script', 'run_script_id', $judgeTask['run_script_id'], $description, $judgeTask['judgetaskid']); + + foreach (glob("$workdir/compile/*") as $compile_file) { + if (!$this->run_command_safe(['cp', '-PRl', $compile_file, $programdir])) { + error("Could not copy program to '$programdir'"); + } + } + + $timelimit_str = implode(':', $timelimit['cpu']) . ',' . implode(':', $timelimit['wall']); + $run_command_parts = [LIBJUDGEDIR . '/testcase_run.sh']; + if (isset($this->options['daemonid'])) { + $run_command_parts[] = '-n'; + $run_command_parts[] = $this->options['daemonid']; + } + array_push($run_command_parts, + $input, + $output, + $timelimit_str, + $passdir, + $run_runpath, + $compare_runpath, + $compare_config['compare_args'] + ); + $this->run_command_safe($run_command_parts, $retval, log_nonzero_exitcode: false); + + // What does the exitcode mean? + if (!isset($this->EXITCODES[$retval])) { + alert('error'); + error("Unknown exitcode ($retval) from testcase_run.sh for s$judgeTask[submitid]"); + } + $result = $this->EXITCODES[$retval]; + + // Try to read metadata from file + $runtime = null; + $metadata = $this->read_metadata($passdir . '/program.meta'); + + if (isset($metadata['time-used']) && array_key_exists($metadata['time-used'], $metadata)) { + $runtime = $metadata[$metadata['time-used']]; + } + + if ($result === 'compare-error') { + $compareMeta = $this->read_metadata($passdir . '/compare.meta'); + $compareExitCode = 'n/a'; + if (isset($compareMeta['exitcode'])) { + $compareExitCode = $compareMeta['exitcode']; + } + if ($combined_run_compare) { + logmsg(LOG_ERR, "comparing failed for combined run/compare script '" . $judgeTask['run_script_id'] . "'"); + $description = 'combined run/compare script ' . $judgeTask['run_script_id'] . ' crashed with exit code ' . $compareExitCode . ", expected one of 42/43"; + $this->disable('run_script', 'run_script_id', $judgeTask['run_script_id'], $description, $judgeTask['judgetaskid']); + } else { + logmsg(LOG_ERR, "comparing failed for compare script '" . $judgeTask['compare_script_id'] . "'"); + logmsg(LOG_ERR, "compare script meta data:\n" . dj_file_get_contents($passdir . '/compare.meta')); + $description = 'compare script ' . $judgeTask['compare_script_id'] . ' crashed with exit code ' . $compareExitCode . ", expected one of 42/43"; + $this->disable('compare_script', 'compare_script_id', $judgeTask['compare_script_id'], $description, $judgeTask['judgetaskid']); + } + return false; + } + + $new_judging_run = [ + 'runresult' => urlencode($result), + 'start_time' => urlencode((string)$startTime), + 'end_time' => urlencode((string)microtime(true)), + 'runtime' => urlencode((string)$runtime), + 'output_run' => $this->rest_encode_file($passdir . '/program.out', $output_storage_limit), + 'output_error' => $this->rest_encode_file($passdir . '/program.err', $output_storage_limit), + 'output_system' => $this->rest_encode_file($passdir . '/system.out', $output_storage_limit), + 'metadata' => $this->rest_encode_file($passdir . '/program.meta', false), + 'output_diff' => $this->rest_encode_file($passdir . '/feedback/judgemessage.txt', $output_storage_limit), + 'hostname' => $this->myhost, + 'testcasedir' => $testcasedir, + 'compare_metadata' => $this->rest_encode_file($passdir . '/compare.meta', false), + ]; + + if (file_exists($passdir . '/feedback/teammessage.txt')) { + $new_judging_run['team_message'] = $this->rest_encode_file($passdir . '/feedback/teammessage.txt', $output_storage_limit); + } + + if ($passLimit > 1) { + $walltime = $metadata['wall-time'] ?? '?'; + logmsg(LOG_INFO, ' ' . ($result === 'correct' ? " \033[0;32m✔\033[0m" : " \033[1;31m✗\033[0m") + . ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result); + } + + if ($result !== 'correct') { + break; + } + if (file_exists($passdir . '/feedback/nextpass.in')) { + $input = $passdir . '/feedback/nextpass.in'; + $nextPass = true; } else { - logmsg(LOG_ERR, "comparing failed for compare script '" . $judgeTask['compare_script_id'] . "'"); - logmsg(LOG_ERR, "compare script meta data:\n" . dj_file_get_contents($passdir . '/compare.meta')); - $description = 'compare script ' . $judgeTask['compare_script_id'] . ' crashed with exit code ' . $compareExitCode . ", expected one of 42/43"; - disable('compare_script', 'compare_script_id', $judgeTask['compare_script_id'], $description, $judgeTask['judgetaskid']); + break; } + } + if ($nextPass) { + $description = 'validator produced more passes than allowed ($passLimit)'; + $this->disable('compare_script', 'compare_script_id', $judgeTask['compare_script_id'], $description, $judgeTask['judgetaskid']); return false; } - $new_judging_run = [ - 'runresult' => urlencode($result), - 'start_time' => urlencode((string)$startTime), - 'end_time' => urlencode((string)microtime(true)), - 'runtime' => urlencode((string)$runtime), - 'output_run' => rest_encode_file($passdir . '/program.out', $output_storage_limit), - 'output_error' => rest_encode_file($passdir . '/program.err', $output_storage_limit), - 'output_system' => rest_encode_file($passdir . '/system.out', $output_storage_limit), - 'metadata' => rest_encode_file($passdir . '/program.meta', false), - 'output_diff' => rest_encode_file($passdir . '/feedback/judgemessage.txt', $output_storage_limit), - 'hostname' => $myhost, - 'testcasedir' => $testcasedir, - 'compare_metadata' => rest_encode_file($passdir . '/compare.meta', false), - ]; - - if (file_exists($passdir . '/feedback/teammessage.txt')) { - $new_judging_run['team_message'] = rest_encode_file($passdir . '/feedback/teammessage.txt', $output_storage_limit); + $ret = true; + if ($result === 'correct') { + // Correct results get reported asynchronously, so we can continue judging in parallel. + $this->reportJudgingRun($judgeTask, $new_judging_run, asynchronous: true); + } else { + // This run was incorrect, only continue with the remaining judge tasks + // if we are told to do so. + $needsMoreWork = $this->reportJudgingRun($judgeTask, $new_judging_run, asynchronous: false); + $ret = (bool)$needsMoreWork; } - if ($passLimit > 1) { + if ($passLimit == 1) { $walltime = $metadata['wall-time'] ?? '?'; - logmsg(LOG_INFO, ' ' . ($result === 'correct' ? " \033[0;32m✔\033[0m" : " \033[1;31m✗\033[0m") + logmsg(LOG_INFO, ' ' . ($result === 'correct' ? " \033[0;32m✔\033[0m" : " \033[1;31m✗\033[0m") . ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result); } - if ($result !== 'correct') { - break; - } - if (file_exists($passdir . '/feedback/nextpass.in')) { - $input = $passdir . '/feedback/nextpass.in'; - $nextPass = true; - } else { - break; - } - } - if ($nextPass) { - $description = 'validator produced more passes than allowed ($passLimit)'; - disable('compare_script', 'compare_script_id', $judgeTask['compare_script_id'], $description, $judgeTask['judgetaskid']); - return false; + // done! + return $ret; } - $ret = true; - if ($result === 'correct') { - // Post result back asynchronously. PHP is lacking multi-threading, so - // we just call ourselves again. - $tmpfile = tempnam(TMPDIR, 'judging_run_'); - file_put_contents($tmpfile, base64_encode(dj_json_encode($new_judging_run))); - $judgedaemon = BINDIR . '/judgedaemon'; - $cmd = $judgedaemon - . ' -e ' . $endpointID - . ' -t ' . $judgeTask['judgetaskid'] - . ' -j ' . $tmpfile - . ' >> /dev/null & '; - shell_exec($cmd); - } else { - // This run was incorrect, only continue with the remaining judge tasks - // if we are told to do so. - $needsMoreWork = request( - sprintf('judgehosts/add-judging-run/%s/%s', urlencode($myhost), - urlencode((string)$judgeTask['judgetaskid'])), - 'POST', - $new_judging_run, - false - ); - $ret = (bool)$needsMoreWork; - } + private function reportJudgingRun(array $judgeTask, array $new_judging_run, bool $asynchronous): ?string + { + $judgeTaskId = $judgeTask['judgetaskid']; - if ($passLimit == 1) { - $walltime = $metadata['wall-time'] ?? '?'; - logmsg(LOG_INFO, ' ' . ($result === 'correct' ? " \033[0;32m✔\033[0m" : " \033[1;31m✗\033[0m") - . ' ...done in ' . $walltime . 's (CPU: ' . $runtime . 's), result: ' . $result); - } + if ($asynchronous && function_exists('pcntl_fork')) { + $pid = pcntl_fork(); + if ($pid === -1) { + logmsg(LOG_WARNING, "Could not fork to report result for jt$judgeTaskId asynchronously, reporting synchronously."); + // Fallback to synchronous reporting by continuing in this process. + } elseif ($pid > 0) { + // Parent process, nothing more to do here. + logmsg(LOG_DEBUG, "Forked a child with PID $pid to report judging run for jt$judgeTaskId."); + return null; + } else { + // Child process: reset signal handlers to default. + pcntl_signal(SIGTERM, SIG_DFL); + pcntl_signal(SIGINT, SIG_DFL); + pcntl_signal(SIGHUP, SIG_DFL); + pcntl_signal(SIGUSR1, SIG_DFL); + + // The child should use its own curl handle to avoid issues with sharing handles + // between processes. + $endpoint = $this->endpoints[$this->endpointID]; + $this->endpoints[$this->endpointID]['ch'] = $this->setup_curl_handle($endpoint['user'], $endpoint['pass']); + } + } elseif ($asynchronous) { + logmsg(LOG_WARNING, "pcntl extension not available, reporting result for jt$judgeTaskId synchronously."); + } - // done! - return $ret; -} + $isChild = isset($pid) && $pid === 0; -function fetchTestcase(string $workdirpath, string $testcase_id, int $judgetaskid, string $testcase_hash): ?array -{ - // Get both in- and output files, only if we didn't have them already. - $tcfile = []; - $bothFilesExist = true; - foreach (['input', 'output'] as $inout) { - $testcasedir = $workdirpath . '/testcase/' . $testcase_id; - if (!is_dir($testcasedir)) { - mkdir($testcasedir, 0755, true); - } - $tcfile[$inout] = $testcasedir . '/' . - $testcase_hash . '.' . - substr($inout, 0, -3); - if (!file_exists($tcfile[$inout])) { - $bothFilesExist = false; + $success = false; + for ($i = 0; $i < 5; $i++) { + if ($i > 0) { + $sleep_ms = 100 + random_int(200, ($i + 1) * 1000); + dj_sleep(0.001 * $sleep_ms); + } + $response = $this->request( + sprintf('judgehosts/add-judging-run/%s/%s', $new_judging_run['hostname'], + urlencode((string)$judgeTaskId)), + 'POST', + $new_judging_run, + false + ); + if ($response !== null) { + logmsg(LOG_DEBUG, "Adding judging run result for jt$judgeTaskId successful."); + $success = true; + break; + } + logmsg(LOG_WARNING, "Failed to report jt$judgeTaskId in attempt #" . ($i + 1) . "."); + } + + if (!$success) { + $message = "Final attempt of uploading jt$judgeTaskId was unsuccessful, giving up."; + if ($isChild) { + error($message); + } else { + warning($message); + return null; + } } + + if ($isChild) { + exit(0); + } + + return $response; } - if ($bothFilesExist) { + + private function fetchTestcase(string $workdirpath, string $testcase_id, int $judgetaskid, string $testcase_hash): ?array + { + // Get both in- and output files, only if we didn't have them already. + $tcfile = []; + $bothFilesExist = true; + foreach (['input', 'output'] as $inout) { + $testcasedir = $workdirpath . '/testcase/' . $testcase_id; + if (!is_dir($testcasedir)) { + mkdir($testcasedir, 0755, true); + } + $tcfile[$inout] = $testcasedir . '/' + . $testcase_hash . '.' . + ($inout == 'input' ? 'in' : 'out'); + if (!file_exists($tcfile[$inout])) { + $bothFilesExist = false; + } + } + if ($bothFilesExist) { + return $tcfile; + } + $content = $this->request(sprintf('judgehosts/get_files/testcase/%s', $testcase_id), 'GET', '', false); + if ($content === null) { + $error = 'Download of testcase failed for case ' . $testcase_id . ', check your problem integrity.'; + logmsg(LOG_ERR, $error); + $this->disable('testcase', 'testcaseid', $testcase_id, $error, $judgetaskid); + return null; + } + $files = dj_json_decode($content); + unset($content); + foreach ($files as $file) { + $filename = $tcfile[$file['filename']]; + file_put_contents($filename, base64_decode($file['content'])); + } + unset($files); + + logmsg(LOG_INFO, " 💾 Fetched new testcase $testcase_id."); return $tcfile; } - $content = request(sprintf('judgehosts/get_files/testcase/%s', $testcase_id), 'GET', '', false); - if ($content === null) { - $error = 'Download of ' . $inout . ' failed for case ' . $testcase_id . ', check your problem integrity.'; - logmsg(LOG_ERR, $error); - disable('testcase', 'testcaseid', $testcase_id, $error, $judgetaskid); - return null; - } - $files = dj_json_decode($content); - unset($content); - foreach ($files as $file) { - $filename = $tcfile[$file['filename']]; - file_put_contents($filename, base64_decode($file['content'])); - } - unset($files); - logmsg(LOG_INFO, " 💾 Fetched new testcase $testcase_id."); - return $tcfile; + private function initsignals(): void + { + pcntl_signal(SIGTERM, [self::class, 'signalHandler']); + pcntl_signal(SIGINT, [self::class, 'signalHandler']); + pcntl_signal(SIGHUP, [self::class, 'signalHandler']); + pcntl_signal(SIGUSR1, [self::class, 'signalHandler']); + } } + +$daemon = new JudgeDaemon(); +$daemon->run(); diff --git a/lib/lib.error.php b/lib/lib.error.php index 6509e9c453..dbb786f0e7 100644 --- a/lib/lib.error.php +++ b/lib/lib.error.php @@ -10,6 +10,7 @@ } // Default verbosity and loglevels: +global $verbose, $loglevel; $verbose = LOG_NOTICE; $loglevel = LOG_DEBUG;