Add contest export script

misc-tools/Makefile

@@ -9,7 +9,8 @@ include $(TOPDIR)/Makefile.global
 TARGETS =
 OBJECTS =
 
-SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest force-passwords
+SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest \
+                  export-contest force-passwords
 
 SUBST_JUDGEHOST = dj_make_chroot dj_run_chroot dj_make_chroot_docker \
                   dj_judgehost_cleanup

misc-tools/dj_utils.py

@@ -30,7 +30,7 @@ def confirm(message: str, default: bool) -> bool:
     return answer == 'y'
 
 
-def parse_api_response(name: str, response: requests.Response):
+def parse_api_response(name: str, response: requests.Response) -> bytes:
     # The connection worked, but we may have received an HTTP error
     if response.status_code >= 300:
         print(response.text)
@@ -44,17 +44,10 @@ def parse_api_response(name: str, response: requests.Response):
     if response.status_code == 204:
         return None
 
-    # We got a successful HTTP response. It worked. Return the full response
-    try:
-        result = json.loads(response.text)
-    except json.decoder.JSONDecodeError:
-        print(response.text)
-        raise RuntimeError(f'Failed to JSON decode the response for API request {name}')
-
-    return result
+    return response.content
 
 
-def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
+def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: bool = True):
     '''Perform an API call to the given endpoint and return its data.
 
     Based on whether `domjudge_webapp_folder_or_api_url` is a folder or URL this
@@ -64,16 +57,18 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
         name (str): the endpoint to call
         method (str): the method to use, GET or PUT are supported
         jsonData (dict): the JSON data to PUT. Only used when method is PUT
+        decode (bool): whether to decode the returned JSON data, default true
 
     Returns:
-        The endpoint contents.
+        The endpoint contents, either as raw bytes or JSON decoded.
 
     Raises:
-        RuntimeError when the response is not JSON or the HTTP status code is non 2xx.
+        RuntimeError when the HTTP status code is non-2xx or the response
+        cannot be JSON decoded.
     '''
 
     if os.path.isdir(domjudge_webapp_folder_or_api_url):
-        return api_via_cli(name, method, {}, {}, jsonData)
+        result = api_via_cli(name, method, {}, {}, jsonData)
     else:
         global ca_check
         url = f'{domjudge_webapp_folder_or_api_url}/{name}'
@@ -86,7 +81,7 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
             if method == 'GET':
                 response = requests.get(url, headers=headers, verify=ca_check, auth=auth)
             elif method == 'PUT':
-                response = requests.put(url, headers=headers, verify=ca_check, json=jsonData, auth=auth)
+                response = requests.put(url, headers=headers, verify=ca_check, auth=auth, json=jsonData)
             else:
                 raise RuntimeError("Method not supported")
         except requests.exceptions.SSLError:
@@ -99,7 +94,16 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
                 return do_api_request(name)
         except requests.exceptions.RequestException as e:
             raise RuntimeError(e)
-    return parse_api_response(name, response)
+        result = parse_api_response(name, response)
+
+    if decode:
+        try:
+            result = json.loads(result)
+        except json.decoder.JSONDecodeError as e:
+            print(result)
+            raise RuntimeError(f'Failed to JSON decode the response for API request {name}')
+
+    return result
 
 
 def upload_file(name: str, apifilename: str, file: str, data: dict = {}):
@@ -121,7 +125,7 @@ def upload_file(name: str, apifilename: str, file: str, data: dict = {}):
     '''
 
     if os.path.isdir(domjudge_webapp_folder_or_api_url):
-        return api_via_cli(name, 'POST', data, {apifilename: file})
+        result = api_via_cli(name, 'POST', data, {apifilename: file})
     else:
         global ca_check
         files = [(apifilename, open(file, 'rb'))]
@@ -141,7 +145,15 @@ def upload_file(name: str, apifilename: str, file: str, data: dict = {}):
                 response = requests.post(
                     url, files=files, headers=headers, data=data, verify=ca_check)
 
-    return parse_api_response(name, response)
+        result = parse_api_response(name, response)
+
+    try:
+        result = json.loads(result)
+    except json.decoder.JSONDecodeError as e:
+        print(result)
+        raise RuntimeError(f'Failed to JSON decode the response for API file upload request {name}')
+
+    return result
 
 
 def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {}, jsonData: dict = {}):
@@ -155,7 +167,7 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {
         jsonData (dict): the JSON data to use. Only used when method is POST or PUT
 
     Returns:
-        The parsed endpoint contents.
+        The raw endpoint contents.
 
     Raises:
         RuntimeError when the command exit code is not 0.
@@ -180,10 +192,14 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {
     command.append(name)
 
     result = subprocess.run(command, capture_output=True)
-    response = result.stdout.decode('ascii')
 
     if result.returncode != 0:
-        print(response)
+        print(
+            f"Command: {command}\nOutput:\n" +
+            result.stdout.decode('utf-8') +
+            result.stderr.decode('utf-8'),
+            file=sys.stderr
+        )
         raise RuntimeError(f'API request {name} failed')
 
-    return json.loads(response)
+    return result.stdout

misc-tools/export-contest.in

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+'''
+export-contest -- Convenience script to export a contest (including metadata,
+teams and problems) from the command line. Defaults to using the CLI interface;
+Specify a DOMjudge API URL as to use that.
+
+Reads credentials from ~/.netrc when using the API.
+
+Part of the DOMjudge Programming Contest Jury System and licensed
+under the GNU GPL. See README and COPYING for details.
+'''
+
+import json
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+sys.path.append('@domserver_libdir@')
+import dj_utils
+
+cid = None
+webappdir = '@domserver_webappdir@'
+
+
+def usage():
+    print(f'Usage: {sys.argv[0]} [<domjudge-api-url>]')
+    exit(1)
+
+
+def api_to_file(endpoint: str, filename: str):
+    print(f"Fetching '{endpoint}' to '{filename}'")
+    data = dj_utils.do_api_request(endpoint, decode=False)
+    with open(filename, 'wb') as f:
+        f.write(data)
+
+    return data
+
+
+def download_submission(submission):
+    d = f'submissions/{submission["id"]}'
+    Path(d).mkdir(parents=True, exist_ok=True)
+    for f in submission['files']:
+        if f['mime'] == 'application/zip':
+            print(f"Downloading '{f['href']}'")
+            data = dj_utils.do_api_request(f['href'], decode=False)
+            with open(f'{d}/files.zip', 'wb') as f:
+                f.write(data)
+
+
+if len(sys.argv) == 1:
+    dj_utils.domjudge_webapp_folder_or_api_url = webappdir
+elif len(sys.argv) == 2:
+    dj_utils.domjudge_webapp_folder_or_api_url = sys.argv[1]
+else:
+    usage()
+
+
+user_data = dj_utils.do_api_request('user')
+if 'admin' not in user_data['roles']:
+    print('Your user does not have the \'admin\' role, can not export.')
+    exit(1)
+
+
+contest_id = 'wf48_finals'
+
+for endpoint in [
+        'accounts',
+        'awards',
+        'balloons',
+        'clarifications',
+        'groups',
+        'judgements',
+        'languages',
+        'organizations',
+        'problems',
+#        'runs',
+        'scoreboard',
+        'submissions',
+        'teams',
+        ]:
+    data = api_to_file(f'contests/{contest_id}/{endpoint}', f'{endpoint}.json')
+    if endpoint == 'submissions':
+        submissions = json.loads(data)
+
+api_to_file(f'contests/{contest_id}/event-feed?stream=false', 'event-feed.ndjson')
+
+with ThreadPoolExecutor(20) as executor:
+    for submission in submissions[:10]:
+        executor.submit(download_submission, submission)

webapp/src/Command/CallApiActionCommand.php

@@ -144,7 +144,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         }
 
         try {
-            $response = null;
+            $response = '';
             $this->dj->withAllRoles(function () use ($input, $data, $files, &$response) {
                 $response = $this->dj->internalApiRequest('/' . $input->getArgument('endpoint'), $input->getOption('method'), $data, $files, true);
             }, $user);
@@ -154,7 +154,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return Command::FAILURE;
         }
 
-        $output->writeln(json_encode($response, JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT));
+        $output->writeln($response);
         return Command::SUCCESS;
     }
 }

webapp/src/Controller/API/ContestController.php

@@ -871,7 +871,7 @@ public function getEventFeedAction(
                     }
 
                     echo Utils::jsonEncode($result) . "\n";
-                    ob_flush();
+                    Utils::ob_flush_if_possible();
                     flush();
                     $lastUpdate = Utils::now();
                     $lastIdSent = $event->getEventid();
@@ -896,7 +896,7 @@ public function getEventFeedAction(
                         # Send keep alive every 10s. Guarantee according to spec is 120s.
                         # However, nginx drops the connection if we don't update for 60s.
                         echo "\n";
-                        ob_flush();
+                        Utils::ob_flush_if_possible();
                         flush();
                         $lastUpdate = $now;
                     }

webapp/src/Service/DOMJudgeService.php

@@ -652,13 +652,15 @@ public function internalApiRequest(string $url, string $method = Request::METHOD
             return null;
         }
 
-        $content = $response->getContent();
-
-        if ($content === '') {
-            return null;
+        if ($response instanceof StreamedResponse) {
+            ob_start(flags: PHP_OUTPUT_HANDLER_REMOVABLE);
+            $response->sendContent();
+            $content = ob_get_clean();
+        } else {
+            $content = $response->getContent();
         }
 
-        return Utils::jsonDecode($content);
+        return $content;
     }
 
     public function getDomjudgeEtcDir(): string

webapp/src/Service/EventLogService.php

@@ -315,20 +315,22 @@
                 $query = ['ids' => $ids];
             }
 
-            $this->dj->withAllRoles(function () use ($query, $url, &$json) {
-                $json = $this->dj->internalApiRequest($url, Request::METHOD_GET, $query);
+            $this->dj->withAllRoles(function () use ($query, $url, &$response) {
+                $response = $this->dj->internalApiRequest($url, Request::METHOD_GET, $query);
             });
 
-            if ($json === null) {
+            if ($response === null || $response === '') {
                 $this->logger->warning(
-                    "EventLogService::log got no JSON data from '%s'", [ $url ]
+                    "EventLogService::log got no data from '%s'", [ $url ]
                 );
                 // If we didn't get data from the API, then that is probably
                 // because this particular data is not visible, for example
                 // because it belongs to an invisible jury team. If we don't
                 // have data, there's also no point in trying to insert
                 // anything in the eventlog table.
                 return;
+            } else {
+                $json = Utils::jsonDecode($response);
             }
         }
 
@@ -484,7 +486,8 @@
                 $url = sprintf('/contests/%s/awards', $contest->getExternalid());
                 $awards = [];
                 $this->dj->withAllRoles(function () use ($url, &$awards) {
-                    $awards = $this->dj->internalApiRequest($url);
+                    $response = $this->dj->internalApiRequest($url);
+                    if ( !empty($response) ) $awards = Utils::jsonDecode($response);
                 });
                 foreach ($awards as $award) {
                     $this->insertEvent($contest, 'awards', $award['id'], $award);
@@ -625,7 +628,8 @@
                 $urlPart = $endpoint === 'contests' ? '' : ('/' . $endpoint);
                 $url = sprintf('/contests/%s%s', $contestId, $urlPart);
                 $this->dj->withAllRoles(function () use ($url, &$data) {
-                    $data = $this->dj->internalApiRequest($url);
+                    $response = $this->dj->internalApiRequest($url);
+                    $data = (empty($response) ? null : Utils::jsonDecode($response));
                 });
 
                 // Get a partial reference to the contest,

webapp/src/Utils/Utils.php

@@ -1023,4 +1023,16 @@
             ini_set('max_execution_time', $minimumMaxExecutionTime);
         }
     }
+
+    /**
+     * Call ob_flush() unless the top-level output buffer does not allow it.
+     */
+    public static function ob_flush_if_possible(): bool
+    {
+        $status = ob_get_status();
+        if ( empty($status) || ($status['flags'] & PHP_OUTPUT_HANDLER_CLEANABLE) ) {
+            return ob_flush();
+        }
+        return false;
+    }
 }

webapp/tests/Unit/Controller/Jury/JuryControllerTestCase.php

@@ -300,6 +300,8 @@ public function testCheckAddEntityAdmin(array $element, array $expected): void
         $this->verifyPageResponse('GET', static::$baseUrl, 200);
         if (static::$add !== '') {
             $response = $this->helperSubmitFields($element);
+            static::assertLessThan(400, $this->client->getInternalResponse()->getStatusCode(),
+                                   $this->client->getInternalResponse()->getContent());
             $this->client->followRedirect();
             foreach ($element as $key => $value) {
                 if (!is_array($value) && !in_array($key, static::$overviewSingleNotShown)) {