Implement submit functionality (#49)

* Implement submission functionality

* Fix broken test automation

Author: kyuridenamida

.travis.yml

@@ -14,7 +14,7 @@ script:
   - autopep8 -r . --diff | tee check_autopep8
   - test ! -s check_autopep8
   - atcoder-tools gen arc050 --without-login
-  - nosetests --with-coverage --cover-package=atcodertools
+  - nosetests tests --exe -v --with-coverage --cover-package=atcodertools
   - codecov
 notifications:
   email: false

atcoder-tools

@@ -3,12 +3,20 @@ import sys
 
 from atcodertools.tools.envgen import main as envgen_main
 from atcodertools.tools.tester import main as tester_main
+from atcodertools.tools.submit import main as submit_main
+
+
+def exit_program(success: bool):
+    sys.exit(0 if success else -1)
+
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2 or sys.argv[1] not in ("gen", "test"):
+    if len(sys.argv) < 2 or sys.argv[1] not in ("gen", "test", "submit"):
         print("Usage:")
         print("{} gen -- to generate workspace".format(sys.argv[0]))
         print("{} test -- to test codes in your workspace".format(sys.argv[0]))
+        print(
+            "{} submit -- to submit a code to the contest system".format(sys.argv[0]))
         sys.exit(-1)
 
     prog = " ".join(sys.argv[:2])
@@ -18,4 +26,7 @@ if __name__ == '__main__':
         envgen_main(prog, args)
 
     if sys.argv[1] == "test":
-        sys.exit(tester_main(prog, args))
+        exit_program(tester_main(prog, args))
+
+    if sys.argv[1] == "submit":
+        exit_program(submit_main(prog, args))

atcodertools/client/atcoder.py

@@ -3,14 +3,15 @@
 import os
 import re
 from http.cookiejar import LWPCookieJar
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 import requests
 from bs4 import BeautifulSoup
 
 from atcodertools.models.contest import Contest
 from atcodertools.models.problem import Problem
 from atcodertools.models.problem_content import ProblemContent, InputFormatDetectionError, SampleDetectionError
+from atcodertools.models.submission import Submission
 
 
 class LoginError(Exception):
@@ -50,6 +51,12 @@ def __call__(cls, *args, **kwargs):
         return cls._instances[cls]
 
 
+def default_credential_supplier() -> Tuple[str, str]:
+    username = input('AtCoder username: ')
+    password = getpass.getpass('AtCoder password: ')
+    return username, password
+
+
 class AtCoderClient(metaclass=Singleton):
 
     def __init__(self):
@@ -60,7 +67,14 @@ def check_logging_in(self):
         resp = self._request(private_url)
         return resp.url == private_url
 
-    def login(self, username=None, password=None, use_local_session_cache=True, save_session_cache=True):
+    def login(self,
+              credential_supplier=None,
+              use_local_session_cache=True,
+              save_session_cache=True):
+
+        if credential_supplier is None:
+            credential_supplier = default_credential_supplier
+
         if use_local_session_cache:
             load_cookie_to(self._session)
             if self.check_logging_in():
@@ -71,11 +85,7 @@ def login(self, username=None, password=None, use_local_session_cache=True, save
 
                 return
 
-        if username is None:
-            username = input('AtCoder username: ')
-
-        if password is None:
-            password = getpass.getpass('AtCoder password: ')
+        username, password = credential_supplier()
 
         resp = self._request("https://arc001.contest.atcoder.jp/login", data={
             'name': username,
@@ -131,31 +141,44 @@ def download_all_contests(self) -> List[Contest]:
         contest_ids = sorted(contest_ids)
         return [Contest(contest_id) for contest_id in contest_ids]
 
-    def submit_source_code(self, contest: Contest, problem: Problem, lang, source):
-        resp = self._request(contest.submission_url())
+    def submit_source_code(self, contest: Contest, problem: Problem, lang: str, source: str) -> Submission:
+        resp = self._request(contest.get_submit_url())
+
         soup = BeautifulSoup(resp.text, "html.parser")
         session_id = soup.find("input", attrs={"type": "hidden"}).get("value")
         task_select_area = soup.find(
             'select', attrs={"id": "submit-task-selector"})
         task_field_name = task_select_area.get("name")
         task_number = task_select_area.find(
             "option", text=re.compile('{} -'.format(problem.get_alphabet()))).get("value")
-
         language_select_area = soup.find(
             'select', attrs={"id": "submit-language-selector-{}".format(task_number)})
         language_field_name = language_select_area.get("name")
         language_number = language_select_area.find(
-            "option", text=re.compile(lang)).get("value")
+            "option", text=lang).get("value")
         postdata = {
             "__session": session_id,
             task_field_name: task_number,
             language_field_name: language_number,
             "source_code": source
         }
-        self._request(
-            contest.get_url(),
+        resp = self._request(
+            contest.get_submit_url(),
             data=postdata,
             method='POST')
+        return Submission.make_submissions_from(resp.text)[0]
+
+    def download_submission_list(self, contest: Contest) -> List[Submission]:
+        submissions = []
+        page_num = 1
+        while True:
+            resp = self._request(contest.get_my_submissions_url(page_num))
+            new_submissions = Submission.make_submissions_from(resp.text)
+            if len(new_submissions) == 0:
+                break
+            submissions += new_submissions
+            page_num += 1
+        return submissions
 
     def _request(self, url: str, method='GET', **kwargs):
         if method == 'GET':

atcodertools/models/contest.py

@@ -1,3 +1,6 @@
+from atcodertools.models.submission import Submission
+
+
 class Contest:
 
     def __init__(self, contest_id):
@@ -7,10 +10,25 @@ def get_id(self):
         return self.contest_id
 
     def get_url(self):
-        return "http://{}.contest.atcoder.jp/".format(self.contest_id)
+        return "https://{}.contest.atcoder.jp/".format(self.contest_id)
 
     def get_problem_list_url(self):
         return "{}assignments".format(self.get_url())
 
-    def submission_url(self):
+    def get_submit_url(self):
         return "{}submit".format(self.get_url())
+
+    def get_my_submissions_url(self, page=1):
+        return "{}submissions/me/{}".format(self.get_url(), page)
+
+    def get_submissions_url(self, submission: Submission):
+        return "{}submissions/{}".format(self.get_url(), submission.submission_id)
+
+    def to_dict(self):
+        return {
+            "contest_id": self.contest_id,
+        }
+
+    @classmethod
+    def from_dict(cls, dic):
+        return Contest(contest_id=dic["contest_id"])

atcodertools/models/problem.py

@@ -16,3 +16,18 @@ def get_url(self):
 
     def get_alphabet(self):
         return self.alphabet
+
+    def to_dict(self):
+        return {
+            "contest": self.contest.to_dict(),
+            "problem_id": self.problem_id,
+            "alphabet": self.alphabet
+        }
+
+    @classmethod
+    def from_dict(cls, dic):
+        return Problem(
+            contest=Contest.from_dict(dic["contest"]),
+            problem_id=dic["problem_id"],
+            alphabet=dic["alphabet"],
+        )

atcodertools/models/submission.py

@@ -0,0 +1,23 @@
+import re
+
+from bs4 import BeautifulSoup
+
+PROB_URL_RE = re.compile(
+    r'"/tasks/([A-Za-z0-9\'~+\-_]+)"')
+SUBMISSION_URL_RE = re.compile(
+    r'"/submissions/([0-9]+)"')
+
+
+class Submission:
+    def __init__(self, problem_id: str, submission_id: int):
+        self.problem_id = problem_id
+        self.submission_id = submission_id
+
+    @staticmethod
+    def make_submissions_from(html: str):
+        soup = BeautifulSoup(html, "html.parser")
+        text = str(soup)
+        submitted_problem_ids = PROB_URL_RE.findall(text)
+        submission_ids = SUBMISSION_URL_RE.findall(text)
+        assert len(submitted_problem_ids) == len(submission_ids)
+        return [Submission(pid, int(sid)) for pid, sid in zip(submitted_problem_ids, submission_ids)]

atcodertools/models/tools/__init__.py

@@ -0,0 +1,40 @@
+import json
+
+from atcodertools.models.problem import Problem
+
+
+class Metadata:
+    def __init__(self, problem: Problem, code_filename: str, sample_in_pattern: str, sample_out_pattern: str, lang: str):
+        self.problem = problem
+        self.code_filename = code_filename
+        self.sample_in_pattern = sample_in_pattern
+        self.sample_out_pattern = sample_out_pattern
+        self.lang = lang
+
+    def to_dict(self):
+        return {
+            "problem": self.problem.to_dict(),
+            "code_filename": self.code_filename,
+            "sample_in_pattern": self.sample_in_pattern,
+            "sample_out_pattern": self.sample_out_pattern,
+            "lang": self.lang,
+        }
+
+    @classmethod
+    def from_dict(cls, dic):
+        return Metadata(
+            problem=Problem.from_dict(dic["problem"]),
+            code_filename=dic["code_filename"],
+            sample_in_pattern=dic["sample_in_pattern"],
+            sample_out_pattern=dic["sample_out_pattern"],
+            lang=dic["lang"],
+        )
+
+    @classmethod
+    def load_from(cls, filename):
+        with open(filename) as f:
+            return cls.from_dict(json.load(f))
+
+    def save_to(self, filename):
+        with open(filename, 'w') as f:
+            json.dump(self.to_dict(), f, indent=1, sort_keys=True)

atcodertools/models/tools/metadata.py

@@ -18,6 +18,8 @@
 from atcodertools.models.problem import Problem
 import logging
 
+from atcodertools.models.tools.metadata import Metadata
+
 script_dir_path = os.path.dirname(os.path.abspath(__file__))
 
 fmt = "%(asctime)s %(levelname)s: %(message)s"
@@ -32,6 +34,10 @@ def extension(lang: str):
     return lang
 
 
+IN_EXAMPLE_FORMAT = "in_{}.txt"
+OUT_EXAMPLE_FORMAT = "out_{}.txt"
+
+
 def prepare_procedure(atcoder_client: AtCoderClient,
                       problem: Problem,
                       workspace_root_path: str,
@@ -68,7 +74,8 @@ def emit_info(text):
         emit_info("No samples.")
     else:
         os.makedirs(workspace_dir_path, exist_ok=True)
-        create_examples(content.get_samples(), workspace_dir_path)
+        create_examples(content.get_samples(), workspace_dir_path,
+                        IN_EXAMPLE_FORMAT, OUT_EXAMPLE_FORMAT)
         emit_info("Created examples.")
 
     code_file_path = os.path.join(
@@ -123,6 +130,16 @@ def emit_info(text):
                 replacement_code_path,
                 code_file_path))
 
+    # Save metadata
+    metadata_path = os.path.join(workspace_dir_path, "metadata.json")
+    Metadata(problem,
+             os.path.basename(code_file_path),
+             IN_EXAMPLE_FORMAT.replace("{}", "*"),
+             OUT_EXAMPLE_FORMAT.replace("{}", "*"),
+             lang,
+             ).save_to(metadata_path)
+    emit_info("Saved metadata to {}".format(metadata_path))
+
 
 def func(argv: Tuple[AtCoderClient, Problem, str, str, str, str]):
     atcoder_client, problem, workspace_root_path, template_code_path, replacement_code_path, lang = argv