fix: adjust time limit

Author: terryyz

bigcodebench/eval/__init__.py

@@ -41,6 +41,7 @@
     swallow_io,
     time_limit,
     safe_environment,
+    TIMEOUT,
 )
 
 
@@ -178,7 +179,7 @@ def untrusted_check(
     gt_time_limit: float = 60
 ) -> Tuple[str, np.ndarray]:
     time_limit = max(min_time_limit, gt_time_limit)
-    timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", 120), time_limit) + 1
+    timeout = max(os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT), time_limit) + 1
     # shared memory objects
     stat = Value("i", _UNKNOWN)
     manager = Manager()

bigcodebench/eval/utils.py

@@ -31,6 +31,8 @@
 import multiprocessing
 from typing import Optional
 
+TIMEOUT=240
+
 @contextlib.contextmanager
 def swallow_subprocess_output():
     """Context manager to swallow stdout and stderr for subprocesses."""

bigcodebench/evaluate.py

@@ -199,11 +199,11 @@ def evaluate(flags):
             def stucking_checker():
                 while remainings:
                     last_size = len(remainings)
-                    time.sleep(120)
+                    time.sleep(240)
                     if last_size != len(remainings) or len(remainings) == 0:
                         continue
                     # Potential stucking
-                    warn("No samples had finished testing in the last 120s")
+                    warn("No samples had finished testing in the last 240s")
                     warn(f"{len(remainings)} samples to be tested: {remainings}")
 
             threading.Thread(target=stucking_checker).start()

bigcodebench/gen/util/__init__.py

@@ -11,6 +11,7 @@
     swallow_io,
     time_limit,
     safe_environment,
+    TIMEOUT,
 )
 
 
@@ -51,7 +52,7 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta
         suite = loader.loadTestsFromTestCase(TestCases)
         test_result = unittest.TestResult()
         start = time.time()
-        with safe_environment(), swallow_io(), time_limit(seconds=120):
+        with safe_environment(), swallow_io(), time_limit(seconds=TIMEOUT):
             suite.run(test_result)
 
         if len(test_result.failures + test_result.errors) > 0:
@@ -68,7 +69,7 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta
 def trusted_check_exec(code, inputs):
     """Check trusted_exec success."""
     try:
-        with time_limit(seconds=120):
+        with time_limit(seconds=TIMEOUT):
             trusted_exec(code, inputs)
     except Exception:
         return False
@@ -83,7 +84,7 @@ def trusted_check(
     max_data_limit: float,
     max_stack_limit: float,
 ):
-    timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", 120) + 1
+    timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", TIMEOUT) + 1
     # shared memory objects
     times = Value("d", -1)
     manager = Manager()