実行時間制限を問題文から取得、ローカルテストに反映する機能を追加 (#254)

* timeoutを問題文から取得するのを追加,
testerのMetadata完全一致を修正

* テストのmetadataをtimeoutに対応

* metadataから制限時間を読むテストを追加

* timeout->timeout_ms

* READMEのtimeout_adjustmentの記述を修正。timeout_adjustmentのテストを追加

Author: chaemon

README.md

@@ -215,6 +215,7 @@ optional arguments:
 
 - `compile_before_testing` テスト前にコンパイルを実行するか否かをTrue/Falseで指定。何も指定しないとFalseとなります。
 - `compile_only_when_diff_detected` テスト前のコンパイルの際、元のソースに変更があった場合のみ実行するかをTrue/Falseで指定。何も指定しないとFalseとなります。
+- `timeout_adjustment=1.2` 問題文に記載された実行時間制限にこの値をかけた秒数がローカルテストの際の実行時間制限になります。例えばatcoderで制限時間2秒の問題は2x1.2=2.4秒となります。atcoderとローカルの実行環境が異なる場合の調整に使用してください。
 
 なお、`compile_before_testing`, `compile_only_when_diff_detected`はいずれもtesterの引数で指定することも可能で、指定した場合はそちらが優先されます。
 
@@ -243,6 +244,7 @@ compile_only_when_diff_detected=true
 [tester]
 compile_before_testing=true
 compile_only_when_diff_detected=true
+timeout_adjustment=1.2
 [etc]
 download_without_login=false
 parallel_download=false

atcodertools/config/tester_config.py

@@ -4,9 +4,11 @@ def __init__(self,
                  compile_before_testing: bool = False,
                  compile_only_when_diff_detected: bool = False,
                  compile_command: str = None,
-                 run_command: str = None
+                 run_command: str = None,
+                 timeout_adjustment: float = 1.0
                  ):
         self.compile_before_testing = compile_before_testing
         self.compile_only_when_diff_detected = compile_only_when_diff_detected
         self.compile_command = compile_command
         self.run_command = run_command
+        self.timeout_adjustment = timeout_adjustment

atcodertools/constprediction/constants_prediction.py

@@ -7,6 +7,7 @@
 from atcodertools.common.judgetype import ErrorType, NormalJudge, DecimalJudge, Judge
 from atcodertools.common.logging import logger
 from atcodertools.constprediction.models.problem_constant_set import ProblemConstantSet
+import math
 
 
 class YesNoPredictionFailedError(Exception):
@@ -19,6 +20,12 @@ def __init__(self, cands):
         self.cands = cands
 
 
+class MultipleLimitCandidatesError(Exception):
+
+    def __init__(self, cands):
+        self.cands = cands
+
+
 class NoDecimalCandidatesError(Exception):
     pass
 
@@ -31,6 +38,7 @@ def __init__(self, cands):
 
 MOD_ANCHORS = ["余り", "あまり", "mod", "割っ", "modulo"]
 DECIMAL_ANCHORS = ["誤差", " error "]
+LIMIT_ANCHORS = ["時間制限", "Time Limit"]
 
 MOD_STRATEGY_RE_LIST = [
     re.compile("([0-9]+).?.?.?で割った"),
@@ -47,6 +55,10 @@ def __init__(self, cands):
     re.compile("1e(-[0-9]+)")
 ]
 
+LIMIT_STRATEGY_RE_LIST = [
+    re.compile("([0-9.]+)\s*sec")
+]
+
 
 def is_mod_context(sentence):
     for kw in MOD_ANCHORS:
@@ -62,6 +74,13 @@ def is_decimal_context(sentence):
     return False
 
 
+def is_limit_context(sentence):
+    for kw in LIMIT_ANCHORS:
+        if kw in sentence:
+            return True
+    return False
+
+
 def predict_modulo(html: str) -> Optional[int]:
     def normalize(sentence):
         return sentence.replace('\\', '').replace("{", "").replace("}", "").replace(",", "").replace(" ", "").replace(
@@ -161,6 +180,32 @@ def normalize(sentence):
     return NormalJudge()
 
 
+def predict_limit(html: str) -> Optional[int]:
+    def normalize(sentence):
+        return sentence.replace('\\', '').replace("{", "").replace("}", "").replace(",", "").replace(" ", "").lower().strip()
+
+    soup = BeautifulSoup(html, "html.parser")
+    sentences = soup.get_text().split("\n")
+    sentences = [normalize(s) for s in sentences if is_limit_context(s)]
+
+    limit_cands = set()
+
+    for s in sentences:
+        for regexp in LIMIT_STRATEGY_RE_LIST:
+            m = regexp.search(s)
+            if m is not None:
+                extracted_val = float(m.group(1))
+                limit_cands.add(extracted_val)
+
+    if len(limit_cands) == 0:
+        return None
+
+    if len(limit_cands) == 1:
+        return math.floor(list(limit_cands)[0] * 1000.0 + 0.5)
+
+    raise MultipleModCandidatesError(limit_cands)
+
+
 def predict_constants(html: str) -> ProblemConstantSet:
     try:
         yes_str, no_str = predict_yes_no(html)
@@ -181,4 +226,11 @@ def predict_constants(html: str) -> ProblemConstantSet:
                        "two or more candidates {} are detected as decimal values".format(e.cands))
         judge = NormalJudge()
 
-    return ProblemConstantSet(mod=mod, yes_str=yes_str, no_str=no_str, judge_method=judge)
+    try:
+        timeout = predict_limit(html)
+    except MultipleLimitCandidatesError as e:
+        logger.warning("limit prediction failed -- "
+                       "two or more candidates {} are detected as limit".format(e.cands))
+        timeout = None
+
+    return ProblemConstantSet(mod=mod, yes_str=yes_str, no_str=no_str, judge_method=judge, timeout=timeout)

atcodertools/constprediction/models/problem_constant_set.py

@@ -8,8 +8,10 @@ def __init__(self,
                  yes_str: str = None,
                  no_str: str = None,
                  judge_method: Judge = None,
+                 timeout: float = None
                  ):
         self.mod = mod
         self.yes_str = yes_str
         self.no_str = no_str
         self.judge_method = judge_method
+        self.timeout = timeout

atcodertools/executils/run_program.py

@@ -30,7 +30,7 @@ def has_stderr(self):
         return len(self.stderr) > 0
 
 
-def run_program(exec_file: str, input_file: str, timeout_sec: int, args=None, current_working_dir: str = None) -> ExecResult:
+def run_program(exec_file: str, input_file: str, timeout_sec: float, args=None, current_working_dir: str = None) -> ExecResult:
     if args is None:
         args = []
     try:

atcodertools/tools/envgen.py

@@ -152,6 +152,7 @@ def emit_info(text):
              config.etc_config.out_example_format.replace("{}", "*"),
              lang,
              constants.judge_method,
+             constants.timeout
              ).save_to(metadata_path)
     emit_info("Saved metadata to {}".format(metadata_path))
 

atcodertools/tools/models/metadata.py

@@ -8,13 +8,14 @@
 class Metadata:
 
     def __init__(self, problem: Problem, code_filename: str, sample_in_pattern: str, sample_out_pattern: str,
-                 lang: Language, judge_method: Judge = NormalJudge()):
+                 lang: Language, judge_method: Judge = NormalJudge(), timeout_ms: int = None):
         self.problem = problem
         self.code_filename = code_filename
         self.sample_in_pattern = sample_in_pattern
         self.sample_out_pattern = sample_out_pattern
         self.lang = lang
         self.judge_method = judge_method
+        self.timeout_ms = timeout_ms
 
     def to_dict(self):
         return {
@@ -24,6 +25,7 @@ def to_dict(self):
             "sample_out_pattern": self.sample_out_pattern,
             "lang": self.lang.name,
             "judge": self.judge_method.to_dict(),
+            "timeout_ms": self.timeout_ms,
         }
 
     @classmethod
@@ -39,13 +41,19 @@ def from_dict(cls, dic):
         else:
             judge_method = NormalJudge()
 
+        if "timeout_ms" in dic:
+            timeout_ms = dic["timeout_ms"]
+        else:
+            timeout_ms = None
+
         return Metadata(
             problem=Problem.from_dict(dic["problem"]),
             code_filename=dic["code_filename"],
             sample_in_pattern=dic["sample_in_pattern"],
             sample_out_pattern=dic["sample_out_pattern"],
             lang=Language.from_name(dic["lang"]),
-            judge_method=judge_method
+            judge_method=judge_method,
+            timeout_ms=timeout_ms
         )
 
     @classmethod

atcodertools/tools/tester.py

@@ -111,7 +111,7 @@ def append(text: str, end='\n'):
     return res
 
 
-def run_for_samples(exec_file: str, sample_pair_list: List[Tuple[str, str]], timeout_sec: int,
+def run_for_samples(exec_file: str, sample_pair_list: List[Tuple[str, str]], timeout_sec: float,
                     judge_method: Judge = NormalJudge(), knock_out: bool = False,
                     skip_io_on_success: bool = False, cwd="./") -> TestSummary:
     success_count = 0
@@ -171,7 +171,7 @@ def validate_sample_pair(in_sample_file, out_sample_file):
         raise IrregularSampleFileError
 
 
-def run_single_test(exec_file, in_sample_file_list, out_sample_file_list, timeout_sec: int, case_num: int,
+def run_single_test(exec_file, in_sample_file_list, out_sample_file_list, timeout_sec: float, case_num: int,
                     judge_method: Judge, cwd: str, judge_program_language: Language) -> bool:
     def single_or_none(lst: List):
         if len(lst) == 1:
@@ -197,7 +197,7 @@ def single_or_none(lst: List):
     return test_summary.success_count == 1 and not test_summary.has_error_output
 
 
-def run_all_tests(exec_file, in_sample_file_list, out_sample_file_list, timeout_sec: int, knock_out: bool,
+def run_all_tests(exec_file, in_sample_file_list, out_sample_file_list, timeout_sec: float, knock_out: bool,
                   skip_stderr_on_success: bool, judge_method: Judge, cwd: str,
                   judge_program_language: Language) -> bool:
     if len(in_sample_file_list) != len(out_sample_file_list):
@@ -303,9 +303,9 @@ def main(prog, args) -> bool:
                         default=".")
 
     parser.add_argument("--timeout", '-t',
-                        help="Timeout for each test cases (sec) [Default] 1",
-                        type=int,
-                        default=1)
+                        help="Timeout for each test cases (sec) [Default] auto",
+                        type=float,
+                        default=None)
 
     parser.add_argument("--knock-out", '-k',
                         help="Stop execution immediately after any example's failure [Default] False",
@@ -374,6 +374,15 @@ def main(prog, args) -> bool:
     with open(args.config, "r") as f:
         config = Config.load(f, {ConfigType.TESTER}, args, lang.name)
 
+    if args.timeout is None:
+        if metadata.timeout_ms is None:
+            logger.info(
+                "timeout_ms is not found in metadata. Default timeout (2.0 sec) is set. ")
+            args.timeout = 2.0
+        else:
+            args.timeout = float(metadata.timeout_ms) / 1000.0
+        args.timeout *= config.tester_config.timeout_adjustment
+
     in_sample_file_list = sorted(
         glob.glob(os.path.join(args.dir, metadata.sample_in_pattern)))
     out_sample_file_list = sorted(

tests/resources/test_atc_env/test_backup/agc029/A/metadata.json

@@ -12,5 +12,6 @@
   "problem_id": "agc029_a"
  },
  "sample_in_pattern": "input_*.txt",
- "sample_out_pattern": "output_*.txt"
+ "sample_out_pattern": "output_*.txt",
+ "timeout_ms": 2000
 }

tests/resources/test_atc_env/test_backup/agc029/B/metadata.json

@@ -12,5 +12,6 @@
   "problem_id": "agc029_b"
  },
  "sample_in_pattern": "input_*.txt",
- "sample_out_pattern": "output_*.txt"
+ "sample_out_pattern": "output_*.txt",
+ "timeout_ms": 2000
 }