fix: change default subset names

Author: terryyz

bigcodebench/data/bigcodebench.py

@@ -16,7 +16,7 @@
 BIGCODEBENCH_HF = "bigcode/bigcodebench"
 BIGCODEBENCH_VERSION = "v0.1.0_hf"
 
-def _ready_bigcodebench_path(subset="", version="default") -> str:
+def _ready_bigcodebench_path(subset="full", version="default") -> str:
     if BIGCODEBENCH_OVERRIDE_PATH:
         return BIGCODEBENCH_OVERRIDE_PATH
 
@@ -39,7 +39,7 @@ def _ready_bigcodebench_path(subset="", version="default") -> str:
 
 
 def get_bigcodebench(
-    err_incomplete=True, subset="", version="default"
+    err_incomplete=True, subset="full", version="default"
     ) -> Dict[str, Dict]:
     """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts.
 
@@ -63,7 +63,7 @@ def get_bigcodebench(
         completeness_check("BigCodeBench", data)
     return data
 
-def get_bigcodebench_hash(subset="", version="default") -> str:
+def get_bigcodebench_hash(subset="full", version="default") -> str:
     """Get the hash of BigCodeBench.
     Returns:
         str: The hash of BigCodeBench

bigcodebench/data/utils.py

@@ -11,8 +11,8 @@
 CACHE_DIR = user_cache_dir("bigcodebench")
 
 
-def get_dataset_metadata(version: str, subset: str=""):
-    extra = "-" + subset.capitalize() if subset else ""
+def get_dataset_metadata(version: str, subset: str="full"):
+    extra = "-" + subset.capitalize() if subset != "full" else ""
     url = f"https://github.com/bigcode-project/bigcodebench-annotation/releases/download/{version}/BigCodeBench{extra}.jsonl.gz"
     cache_path = os.path.join(CACHE_DIR, f"BigCodeBench{extra}-{version}.jsonl")
     return url, cache_path

bigcodebench/evaluate.py

@@ -287,7 +287,7 @@ def main():
     parser.add_argument(
         "--split", required=True, type=str, choices=["complete", "instruct"]
     )
-    parser.add_argument("--subset", default="", choices=["", "hard"], type=str)
+    parser.add_argument("--subset", default="full", choices=["full", "hard"], type=str)
     parser.add_argument("--samples", required=True, type=str)
     parser.add_argument("--parallel", default=None, type=int)
     parser.add_argument("--min-time-limit", default=1, type=float)

bigcodebench/generate.py

@@ -24,7 +24,7 @@ def codegen(
     id_range=None,
     resume=True,
 ):
-    extra = "-" + subset.capitalize() if subset else ""
+    extra = "-" + subset.capitalize()
     with Progress(
         TextColumn(f"BigCodeBench--{split} ({extra}) •" + "[progress.percentage]{task.percentage:>3.0f}%"),
         BarColumn(),
@@ -108,7 +108,7 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--model", required=True, type=str)
     parser.add_argument("--split", required=True, type=str)
-    parser.add_argument("--subset", default="", type=str)
+    parser.add_argument("--subset", default="full", type=str)
     parser.add_argument("--save_path", default=None, type=str)
     parser.add_argument("--bs", default=1, type=int)
     parser.add_argument("--n_samples", default=1, type=int)
@@ -125,7 +125,7 @@ def main():
 
 
     assert args.split in ["complete", "instruct"], f"Invalid split {args.split}"
-    assert args.subset in ["", "hard"], f"Invalid subset {args.subset}"
+    assert args.subset in ["full", "hard"], f"Invalid subset {args.subset}"
     assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"]
 
     if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\