WIP add export-contest script - Jaap

Author: DOMjudge team

misc-tools/Makefile

@@ -9,7 +9,8 @@ include $(TOPDIR)/Makefile.global
 TARGETS =
 OBJECTS =
 
-SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest
+SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest \
+                  export-contest
 
 SUBST_JUDGEHOST = dj_make_chroot dj_run_chroot dj_make_chroot_docker \
                   dj_judgehost_cleanup

misc-tools/dj_utils.py

@@ -44,17 +44,10 @@ def parse_api_response(name: str, response: requests.Response):
     if response.status_code == 204:
         return None
 
-    # We got a successful HTTP response. It worked. Return the full response
-    try:
-        result = json.loads(response.text)
-    except json.decoder.JSONDecodeError as e:
-        print(response.text)
-        raise RuntimeError(f'Failed to JSON decode the response for API request {name}')
-
-    return result
+    return response.text
 
 
-def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
+def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: bool = True):
     '''Perform an API call to the given endpoint and return its data.
 
     Based on whether `domjudge_webapp_folder_or_api_url` is a folder or URL this
@@ -64,16 +57,18 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
         name (str): the endpoint to call
         method (str): the method to use, GET or PUT are supported
         jsonData (dict): the JSON data to PUT. Only used when method is PUT
+        decode (bool): whether to decode the returned JSON data, default true
 
     Returns:
         The endpoint contents.
 
     Raises:
-        RuntimeError when the response is not JSON or the HTTP status code is non 2xx.
+        RuntimeError when the HTTP status code is non-2xx or the response
+        cannot be JSON decoded.
     '''
 
     if os.path.isdir(domjudge_webapp_folder_or_api_url):
-        return api_via_cli(name, method, {}, {}, jsonData)
+        result = api_via_cli(name, method, {}, {}, jsonData)
     else:
         global ca_check
         url = f'{domjudge_webapp_folder_or_api_url}/{name}'
@@ -97,7 +92,17 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}):
                 return do_api_request(name)
         except requests.exceptions.RequestException as e:
             raise RuntimeError(e)
-    return parse_api_response(name, response)
+        result = parse_api_response(name, response)
+
+    if decode:
+        try:
+            result = json.loads(result)
+        except json.decoder.JSONDecodeError as e:
+            print(result)
+            raise RuntimeError(f'Failed to JSON decode the response for API request {name}')
+
+    return result
+
 
 def upload_file(name: str, apifilename: str, file: str, data: dict = {}):
     '''Upload the given file to the API at the given path with the given name.
@@ -118,7 +123,7 @@ def upload_file(name: str, apifilename: str, file: str, data: dict = {}):
     '''
 
     if os.path.isdir(domjudge_webapp_folder_or_api_url):
-        return api_via_cli(name, 'POST', data, {apifilename: file})
+        response = api_via_cli(name, 'POST', data, {apifilename: file})
     else:
         global ca_check
         files = [(apifilename, open(file, 'rb'))]
@@ -152,7 +157,7 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {
         jsonData (dict): the JSON data to use. Only used when method is POST or PUT
 
     Returns:
-        The parsed endpoint contents.
+        The endpoint contents.
 
     Raises:
         RuntimeError when the command exit code is not 0.
@@ -183,4 +188,4 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {
         print(response)
         raise RuntimeError(f'API request {name} failed')
 
-    return json.loads(response)
+    return response

misc-tools/export-contest.in

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+'''
+export-contest -- Convenience script to export a contest (including metadata,
+teams and problems) from the command line. Defaults to using the CLI interface;
+Specify a DOMjudge API URL as to use that.
+
+Reads credentials from ~/.netrc when using the API.
+
+Part of the DOMjudge Programming Contest Jury System and licensed
+under the GNU GPL. See README and COPYING for details.
+'''
+
+import json
+import sys
+# from concurrent.futures import ThreadPoolExecutor
+# from multiprocessing import Pool
+from pathlib import Path
+
+sys.path.append('@domserver_libdir@')
+import dj_utils
+
+cid = None
+webappdir = '@domserver_webappdir@'
+
+
+def usage():
+    print(f'Usage: {sys.argv[0]} [<domjudge-api-url>]')
+    exit(1)
+
+
+def api_to_file(endpoint: str, filename: str):
+    print(f"Fetching '{endpoint}' to '{filename}'")
+    data = dj_utils.do_api_request(endpoint, decode=False)
+    with open(filename, 'w') as f:
+        f.write(data)
+
+    return data
+
+
+def download_submission(submission):
+    d = f'submissions/{submission["id"]}'
+    Path(d).mkdir(parents=True, exist_ok=True)
+    for f in submission['files']:
+        if f['mime'] == 'application/zip':
+            print(f"Downloading '{f['href']}'")
+            data = dj_utils.do_api_request(f['href'], decode=False)
+            with open(f'{d}/files.zip', 'w') as f:
+                f.write(data)
+            break
+
+
+if len(sys.argv) == 1:
+    dj_utils.domjudge_webapp_folder_or_api_url = webappdir
+elif len(sys.argv) == 2:
+    dj_utils.domjudge_webapp_folder_or_api_url = sys.argv[1]
+else:
+    usage()
+
+
+user_data = dj_utils.do_api_request('user')
+if 'admin' not in user_data['roles']:
+    print('Your user does not have the \'admin\' role, can not export.')
+    exit(1)
+
+
+contest_id = 'wf48_systest2'
+
+for endpoint in [
+        'accounts',
+        'awards',
+        'balloons',
+        'clarifications',
+        'groups',
+        'judgements',
+        'languages',
+        'organizations',
+        'problems',
+        'runs',
+        'scoreboard',
+        'submissions',
+        'teams',
+        ]:
+    data = api_to_file(f'contests/{contest_id}/{endpoint}', f'{endpoint}.json')
+    if endpoint == 'submissions':
+        submissions = json.loads(data)
+
+api_to_file(f'contests/{contest_id}/event-feed?stream=false', 'event-feed.ndjson')
+
+
+for submission in submissions:
+    download_submission(submission)
+
+# with Pool(processes=10) as pool:
+#     result = pool.map_async(download_submission, submissions)
+# 
+# print(result)
+# result.wait()
+
+# with ThreadPoolExecutor(20) as executor:
+#     for submission in submissions:
+#         executor.submit(download_submission, submission)