fix: change subset namings

Author: terryyz

bigcodebench/data/bigcodebench.py

@@ -25,7 +25,7 @@ def _ready_bigcodebench_path(subset="", version="default") -> str:
         BIGCODEBENCH_VERSION, subset
     )
 
-    extra = "-subset" if subset else ""
+    extra = "-" + subset if subset else ""
 
     try:
         dataset = load_dataset(BIGCODEBENCH_HF+extra, split=BIGCODEBENCH_VERSION)
@@ -39,7 +39,7 @@ def _ready_bigcodebench_path(subset="", version="default") -> str:
 
 
 def get_bigcodebench(
-    err_incomplete=True, subset="full", version="default"
+    err_incomplete=True, subset="", version="default"
     ) -> Dict[str, Dict]:
     """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts.
 

bigcodebench/evaluate.py

@@ -124,8 +124,8 @@ def evaluate(flags):
         assert flags.samples.endswith(".jsonl")
         result_path = flags.samples.replace(".jsonl", "_eval_results.json")
 
-    problems = get_bigcodebench(hard=flags.hard)
-    dataset_hash = get_bigcodebench_hash(hard=flags.hard)
+    problems = get_bigcodebench(subset=flags.subset)
+    dataset_hash = get_bigcodebench_hash(subset=flags.subset)
 
     if not flags.no_gt:
         expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit)
@@ -287,7 +287,7 @@ def main():
     parser.add_argument(
         "--split", required=True, type=str, choices=["complete", "instruct"]
     )
-    parser.add_argument("--hard", action="store_true")
+    parser.add_argument("--subset", default="", choices=["", "hard"], type=str)
     parser.add_argument("--samples", required=True, type=str)
     parser.add_argument("--parallel", default=None, type=int)
     parser.add_argument("--min-time-limit", default=1, type=float)

bigcodebench/generate.py

@@ -125,6 +125,7 @@ def main():
 
 
     assert args.split in ["complete", "instruct"], f"Invalid split {args.split}"
+    assert args.subset in ["", "hard"], f"Invalid subset {args.subset}"
     assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"]
 
     if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\