rename subset

Author: terryyz

README.md

@@ -92,7 +92,7 @@ To generate code samples from a model, you can use the following command:
 ```shell
 bigcodebench.generate \
     --model [model_name] \
-    --subset [c2c|nl2c] \
+    --subset [complete|instruct] \
     --greedy \
     --bs [bs] \
     --temperature [temp] \
@@ -102,12 +102,12 @@ bigcodebench.generate \
     --tp [gpu_number]
 ```
 >
-The generated code samples will be stored in a file named `[model_name]--bigcodebench-[nl2c|c2c]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples:
+The generated code samples will be stored in a file named `[model_name]--bigcodebench-[instruct|complete]--[backend]-[temp]-[n_samples].jsonl`. Alternatively, you can use the following command to utilize our pre-built docker images for generating code samples:
 >
 ```shell
 docker run --gpus '"device=$CUDA_VISIBLE_DEVICES"' -v $(pwd):/bigcodebench -t terryzho/bigcodebench-generate-cu11:latest \
     --model [model_name] \ 
-    --subset [c2c|nl2c] \
+    --subset [complete|instruct] \
     --greedy \
     --bs [bs] \   
     --temperature [temp] \
@@ -186,9 +186,9 @@ You are strongly recommended to use a sandbox such as [docker](https://docs.dock
 
 ```shell
 # mount the current directory to the container
-docker run -v $(pwd):/bigcodebench terryzho/bigcodebench-evaluate:latest --subset [c2c|nl2c] --samples samples.jsonl
+docker run -v $(pwd):/bigcodebench terryzho/bigcodebench-evaluate:latest --subset [complete|instruct] --samples samples.jsonl
 # ...Or locally ⚠️
-bigcodebench.evaluate --subset [c2c|nl2c] --samples samples.jsonl
+bigcodebench.evaluate --subset [complete|instruct] --samples samples.jsonl
 ```
 
 ...Or if you want to try it locally regardless of the risks ⚠️:
@@ -202,7 +202,7 @@ pip install -r https://raw.githubusercontent.com/bigcode-project/bigcodebench-an
 Then, run the evaluation:
 
 ```shell
-bigcodebench.evaluate --subset [c2c|nl2c] --samples samples.jsonl
+bigcodebench.evaluate --subset [complete|instruct] --samples samples.jsonl
 ```
 
 > [!Tip]

bigcodebench/evaluate.py

@@ -242,7 +242,7 @@ def stucking_checker():
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--subset", required=True, type=str, choices=["c2c", "nl2c"]
+        "--subset", required=True, type=str, choices=["complete", "instruct"]
     )
     parser.add_argument("--samples", required=True, type=str)
     parser.add_argument("--parallel", default=None, type=int)

bigcodebench/generate.py

@@ -33,8 +33,8 @@ def codegen(
 
         dataset = get_bigcodebench()
 
-        if model.is_direct_completion() and subset == "nl2c":
-            raise Exception("Base model does not support direct completion for NL2Code tasks")
+        if model.is_direct_completion() and subset == "instruct":
+            raise Exception("Base model does not support direct completion for instructode tasks")
 
         # create save_path if it doesn't exist, e.g., a/b.jsonl
         dirname = os.path.dirname(save_path)
@@ -70,7 +70,7 @@ def codegen(
             sidx = n_samples - nsamples
             while sidx < n_samples:
                 prompt = task["prompt"]
-                if subset == "nl2code":
+                if subset == "instructode":
                     prompt = task["instruction"]
                 if strip_newlines:
                     prompt = prompt.strip("\n")
@@ -119,7 +119,7 @@ def main():
     args = parser.parse_args()
 
 
-    assert args.subset in ["c2c", "nl2c"], f"Invalid subset {args.subset}"
+    assert args.subset in ["complete", "instruct"], f"Invalid subset {args.subset}"
     assert args.backend in ["vllm", "hf", "openai", "mistral", "anthropic", "google"]
 
     if args.greedy and (args.temperature != 0 or args.bs != 1 or args.n_samples != 1)\

run.sh

@@ -5,7 +5,7 @@ BACKEND=openai
 TEMP=0
 N_SAMPLES=1
 NUM_GPU=1
-SUBSET=nl2c
+SUBSET=instruct
 if [[ $MODEL == *"/"* ]]; then
   ORG=$(echo $MODEL | cut -d'/' -f1)--
   BASE_MODEL=$(echo $MODEL | cut -d'/' -f2)