Merge pull request #44 from loubnabnl/integrate_multipl-e

Integrate MultiPL-E

Author: loubnabnl

Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+COPY . /app
+
+WORKDIR /app
+
+RUN test -f /app/generations.json && rm /app/generations.json || true
+
+RUN pip3 install .
+
+CMD ["python3", "main.py"]

Dockerfile-multiple

@@ -0,0 +1,56 @@
+FROM ubuntu:22.04
+RUN apt-get update -yqq && apt-get install -yqq curl build-essential python3-pip python3-tqdm
+RUN apt-get install racket -yqq
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+RUN apt-get install -yqq \
+    default-jdk-headless \
+    golang-go \
+    php-cli \
+    ruby \
+    lua5.3 \
+    r-base \
+    rustc \
+    scala
+
+RUN apt-get install -yqq libtest-deep-perl 
+RUN apt-get install -yqq wget 
+
+# JS/TS
+RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - 
+RUN apt-get install -y nodejs
+RUN npm install -g typescript
+
+# Dlang
+RUN wget https://netcologne.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list
+RUN apt-get update --allow-insecure-repositories
+RUN apt-get -y --allow-unauthenticated install --reinstall d-apt-keyring
+RUN apt-get update && apt-get install -yqq dmd-compiler dub
+
+# C#
+RUN apt install gnupg ca-certificates
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+RUN echo "deb https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list
+RUN apt update
+RUN apt install -yqq mono-devel
+
+# Post-processing
+
+# Julia
+RUN curl https://julialang-s3.julialang.org/bin/linux/x64/1.8/julia-1.8.2-linux-x86_64.tar.gz | tar xz
+ENV PATH="/julia-1.8.2/bin:${PATH}"
+# Swift
+RUN curl https://download.swift.org/swift-5.7-release/ubuntu2204/swift-5.7-RELEASE/swift-5.7-RELEASE-ubuntu22.04.tar.gz | tar xz
+ENV PATH="/swift-5.7-RELEASE-ubuntu22.04/usr/bin:${PATH}"
+# Javatuples
+RUN mkdir /usr/multiple && wget https://repo.mavenlibs.com/maven/org/javatuples/javatuples/1.2/javatuples-1.2.jar -O /usr/multiple/javatuples-1.2.jar
+# Luaunit
+RUN apt-get update -yqq && apt-get install -yqq lua-unit
+
+# Standard requirements
+COPY . /app
+WORKDIR /app
+RUN test -f /app/generations.json && rm /app/generations.json || true
+
+RUN pip3 install .
+CMD ["python3", "main.py"]

README.md

@@ -110,6 +110,49 @@ Below is an example, be mind of specifying arguments proper to the task you are
 ```bash
 accelerate launch  main.py   --tasks mbpp  --allow_code_execution  --load_generations_path generations.json  --model incoder-temperature-08
 ```
+## Docker containers
+For safety, we provide a Dockerfiles to do the execution inside a docker container. To do that, first, do the generation on your machine and save them in generations.json by adding the flag --generation_only to the command. Then build the docker container and run the evaluation inside it.
+
+### Building  Docker image
+Here's how to build a docker image for the evaluation harness:
+```bash
+$ sudo make DOCKERFILE=Dockerfile  all
+```
+This creates an image called `evaluation-harness`, and runs a test on it. To skip the test remove `all` form the command.
+
+If you want to evaluate on MultiPL-E, we have a different Dockerfile since it requires more dependencies, use:
+```bash
+$ sudo make DOCKERFILE=Dockerfile-multiple all
+```
+This creates an image called `evaluation-harness-multiple`.
+
+### Evaluating inside a container
+Suppose you generated text with the `bigcode/santacoder` model and saved it in `generations.json` with:
+```bash
+accelerate launch  main.py \
+    --model bigcode/santacoder  \
+    --tasks multiple-py  \
+    --max_length_generation 650 \
+    --temperature 0.8   \
+    --do_sample True  \
+    --n_samples 200  \
+    --batch_size 200  \
+    --trsut_remote_code \
+    --generation_only \
+    --save_generations \
+    --save_generations_path generations_py.json
+```
+
+To run the container (here from image `evaluation-harness`) to evaluate on `generations.json`, or another file mount it with `-v`, specify `n_samples` and allow code execution with `--allow_code_execution` (and add the number of problems `--limit`  if it was used during generation):
+```bash
+$ sudo docker run -v $(pwd)/generations_py.json:/app/generations_py.json:ro -it evaluation-harness-multiple python3 main.py \
+    --model bigcode/santacoder \
+    --tasks multiple-py \
+    --load_generations_path /app/generations_py.json \
+    --allow_code_execution  \
+    --temperature 0.8 \
+    --n_samples 200
+```
 
 ## Implementing new tasks
 To implement a new task in this evaluation harness, see the guide in [`docs/guide`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/guide.md). The are also contribution guidelines in this [`CONTRIBUTING.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/CONTRIBUTING.md)

finetuning/APPS/apps_dataset.py

@@ -10,7 +10,9 @@ class APPSBaseDataset(torch.utils.data.Dataset):
     def __init__(self, dataset, max_tokens, tokenizer_path):
         self.dataset = dataset
         self.max_tokens = max_tokens
-        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_auth_token=True)
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            tokenizer_path, use_auth_token=True
+        )
         self.samples = []  # Should be set in initialize()
 
         self.initialize(self.tokenizer)

finetuning/APPS/apps_train.py

@@ -4,17 +4,12 @@
 
 import argparse
 import os
-import torch
 
+import torch
 from apps_dataset import APPSBaseDataset
 from datasets import load_dataset
-from transformers import (
-    AutoModelForCausalLM,
-    Trainer,
-    TrainingArguments,
-    logging,
-    set_seed,
-)
+from transformers import (AutoModelForCausalLM, Trainer, TrainingArguments,
+                          logging, set_seed)
 
 
 def get_args():
@@ -59,22 +54,20 @@ def run_training(args, train_data, val_data):
     training_args = TrainingArguments(
         output_dir=args.output_dir,
         dataloader_drop_last=True,
-        evaluation_strategy = "steps",
+        evaluation_strategy="steps",
         num_train_epochs=args.num_epochs,
-        max_steps = args.max_steps,
-        eval_steps = args.eval_freq,
+        max_steps=args.max_steps,
+        eval_steps=args.eval_freq,
         save_steps=args.save_freq,
         logging_steps=args.log_freq,
-
         per_device_train_batch_size=args.batch_size,
         per_device_eval_batch_size=args.batch_size,
         learning_rate=args.learning_rate,
         lr_scheduler_type=args.lr_scheduler_type,
-        warmup_steps = args.num_warmup_steps,
+        warmup_steps=args.num_warmup_steps,
         gradient_accumulation_steps=args.gradient_accumulation_steps,
         weight_decay=args.weight_decay,
         fp16=args.fp16,
-
         run_name="apps-train",
         report_to="wandb",
     )
@@ -99,8 +92,14 @@ def main(args):
     dataset.shuffle(seed=args.seed)
     data = get_dataset(dataset, args)
     train_size = int(0.95 * len(data))
-    train_data, val_data = torch.utils.data.random_split(data, [train_size, len(data) - train_size], generator=torch.Generator().manual_seed(args.seed))
-    print(f"size of training data {len(train_data)}\nsize of validation data {len(val_data)}")
+    train_data, val_data = torch.utils.data.random_split(
+        data,
+        [train_size, len(data) - train_size],
+        generator=torch.Generator().manual_seed(args.seed),
+    )
+    print(
+        f"size of training data {len(train_data)}\nsize of validation data {len(val_data)}"
+    )
     run_training(args, train_data, val_data)
 
 

finetuning/Code-to-text/train.py

@@ -1,19 +1,15 @@
 import argparse
 
 from datasets import load_dataset
-
-from transformers import (
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
-    Trainer,
-    TrainingArguments,
-    set_seed,
-)
+from transformers import (AutoModelForSequenceClassification, AutoTokenizer,
+                          Trainer, TrainingArguments, set_seed)
 
 
 def get_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--model_ckpt", type=str, default="microsoft/unixcoder-base-nine")
+    parser.add_argument(
+        "--model_ckpt", type=str, default="microsoft/unixcoder-base-nine"
+    )
     parser.add_argument("--language", type=str, default="Python")
     parser.add_argument("--max_length", type=int, default=1024)
     parser.add_argument("--num_epochs", type=int, default=5)
@@ -40,7 +36,9 @@ def main():
     print("Loading tokenizer and model")
     tokenizer = AutoTokenizer.from_pretrained(args.model_ckpt)
     tokenizer.pad_token = tokenizer.eos_token
-    model = AutoModelForSequenceClassification.from_pretrained(args.model_ckpt, num_labels=2)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        args.model_ckpt, num_labels=2
+    )
     model.config.pad_token_id = model.config.eos_token_id
 
     if args.freeze:
@@ -49,13 +47,20 @@ def main():
 
     def tokenize(example):
         if args.language == "Python":
-            #remove docstring from code
+            # remove docstring from code
             chunks = example["code"].split('"""')
             code = chunks[0].strip() + chunks[2]
         else:
             code = example["code"]
-        inputs = tokenizer(code, padding="max_length", truncation=True, max_length=args.max_length)  
-        labels = tokenizer(example["docstring"], padding="max_length", truncation=True, max_length=args.max_length).input_ids
+        inputs = tokenizer(
+            code, padding="max_length", truncation=True, max_length=args.max_length
+        )
+        labels = tokenizer(
+            example["docstring"],
+            padding="max_length",
+            truncation=True,
+            max_length=args.max_length,
+        ).input_ids
         labels_with_ignore_index = []
         for labels_example in labels:
             labels_example = [label if label != 0 else -100 for label in labels_example]
@@ -99,10 +104,11 @@ def tokenize(example):
 
     print("Training...")
     trainer.train()
-    
+
     # push the model to the Hugging Face hub
     if args.push_to_hub:
         model.push_to_hub(args.model_hub_name)
 
+
 if __name__ == "__main__":
     main()

finetuning/CodeClone/train.py

@@ -3,22 +3,17 @@
 
 import numpy as np
 from datasets import ClassLabel, load_dataset
-
 from evaluate import load
-from transformers import (
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
-    DataCollatorWithPadding,
-    Trainer,
-    TrainerCallback,
-    TrainingArguments,
-    set_seed,
-)
+from transformers import (AutoModelForSequenceClassification, AutoTokenizer,
+                          DataCollatorWithPadding, Trainer, TrainerCallback,
+                          TrainingArguments, set_seed)
 
 
 def get_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--model_ckpt", type=str, default="microsoft/unixcoder-base-nine")
+    parser.add_argument(
+        "--model_ckpt", type=str, default="microsoft/unixcoder-base-nine"
+    )
     parser.add_argument("--max_length", type=int, default=1024)
     parser.add_argument("--num_epochs", type=int, default=5)
     parser.add_argument("--batch_size", type=int, default=6)
@@ -52,7 +47,9 @@ def __init__(self, trainer) -> None:
     def on_epoch_end(self, args, state, control, **kwargs):
         if control.should_evaluate:
             control_copy = deepcopy(control)
-            self._trainer.evaluate(eval_dataset=self._trainer.train_dataset, metric_key_prefix="train")
+            self._trainer.evaluate(
+                eval_dataset=self._trainer.train_dataset, metric_key_prefix="train"
+            )
             return control_copy
 
 
@@ -61,21 +58,28 @@ def main():
     set_seed(args.seed)
 
     ds = load_dataset("code_x_glue_cc_clone_detection_big_clone_bench")
-    labels = ClassLabel(num_classes = 2, names=[True, False])
+    labels = ClassLabel(num_classes=2, names=[True, False])
     ds = ds.cast_column("label", labels)
 
     print("Loading tokenizer and model")
     tokenizer = AutoTokenizer.from_pretrained(args.model_ckpt)
     tokenizer.pad_token = tokenizer.eos_token
-    model = AutoModelForSequenceClassification.from_pretrained(args.model_ckpt, num_labels=2)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        args.model_ckpt, num_labels=2
+    )
     model.config.pad_token_id = model.config.eos_token_id
 
     if args.freeze:
         for param in model.roberta.parameters():
             param.requires_grad = False
 
     def tokenize(example):
-        inputs = tokenizer(example["func1"], example["func2"], truncation=True, max_length=args.max_length)  
+        inputs = tokenizer(
+            example["func1"],
+            example["func2"],
+            truncation=True,
+            max_length=args.max_length,
+        )
         return {
             "input_ids": inputs["input_ids"],
             "attention_mask": inputs["attention_mask"],
@@ -121,10 +125,11 @@ def tokenize(example):
 
     result = trainer.evaluate(eval_dataset=tokenized_datasets["test"])
     print(f"Evaluation accuracy on the test set: {result['eval_accuracy']}")
-    
+
     # push the model to the Hugging Face hub
     if args.push_to_hub:
         model.push_to_hub(args.model_hub_name)
 
+
 if __name__ == "__main__":
     main()