fix: change dataset to subset

Author: terryyz

bigcodebench/eval/__init__.py

@@ -106,7 +106,6 @@ def is_floats(x) -> bool:
 
 
 def unsafe_execute(
-    dataset: str,
     entry_point: str,
     code: str,
     test_code: str,
@@ -168,7 +167,6 @@ def unsafe_execute(
 
 
 def untrusted_check(
-    dataset: str,
     code: str,
     test_code: str,
     entry_point: str,
@@ -185,7 +183,6 @@ def untrusted_check(
     p = multiprocessing.Process(
         target=unsafe_execute,
         args=(
-            dataset,
             entry_point,
             code,
             test_code,
@@ -217,7 +214,6 @@ def untrusted_check(
 
 
 def evaluate_files(
-    dataset: str,
     files: List[str],
     inputs: List,
     entry_point: str,
@@ -230,7 +226,6 @@ def evaluate_files(
     for file in files:
         code = open(file, "r").read()
         stat, det = untrusted_check(
-            dataset,
             code,
             inputs,
             entry_point,

bigcodebench/evaluate.py

@@ -62,7 +62,6 @@ def get_groundtruth(problems, hashcode, check_gt_only):
     return expected_time
 
 def check_correctness(
-    dataset: str,
     completion_id: int,
     problem: Dict[str, Any],
     solution: str,
@@ -77,7 +76,6 @@ def check_correctness(
         "solution": solution,
     }
     ret["base"] = untrusted_check(
-        dataset,
         solution,
         problem["test"],
         problem["entry_point"],
@@ -119,10 +117,9 @@ def evaluate(flags):
 
         results = compatible_eval_result(results)
     else:
-        if flags.dataset == "bigcodebench":
-            problems = get_bigcodebench()
-            dataset_hash = get_bigcodebench_hash()       
-            expected_time = get_groundtruth(problems, dataset_hash, flags.check_gt_only)
+        problems = get_bigcodebench()
+        dataset_hash = get_bigcodebench_hash()       
+        expected_time = get_groundtruth(problems, dataset_hash, flags.check_gt_only)
 
         if flags.check_gt_only:
             return
@@ -157,7 +154,6 @@ def evaluate(flags):
                     solution = problems[task_id]["prompt_wo_doc"] + "\n    pass\n" + solution
                 remainings.add(sample["_identifier"])
                 args = (
-                    flags.dataset,
                     completion_id[task_id],
                     problems[task_id],
                     solution,
@@ -219,7 +215,7 @@ def stucking_checker():
         for k in [1, 5, 10, 25, 100]
         if total.min() >= k
     }
-    cprint(f"{flags.dataset}", "green")
+    cprint(f"BigCodeBench-{flags.subset}", "green")
     for k, v in pass_at_k.items():
         cprint(f"{k}:\t{v:.3f}", "green")
 
@@ -246,7 +242,7 @@ def stucking_checker():
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--dataset", required=True, type=str, choices=["bigcodebench"]
+        "--subset", required=True, type=str, choices=["c2c", "nl2c"]
     )
     parser.add_argument("--samples", required=True, type=str)
     parser.add_argument("--parallel", default=None, type=int)