Merge pull request #30 from OfirArviv/main

Add Safety evaluation reproducible readme

Author: perlitz

evaluation/README.md

@@ -4,8 +4,8 @@
 
 initialize and environment, sometimes `conda` makes it easier to handle cuda installations:
 ```bash
-conda create -n bamba_vllm python=3.11 -y
-conda activate bamba_vllm
+conda create -n bamba python=3.11 -y
+conda activate bamba
 ```
 
 Instal cuda toolkit:
@@ -23,14 +23,19 @@ pip install "causal-conv1d @ git+https://github.com/Dao-AILab/[email protected]
 pip install git+https://github.com/fabianlim/transformers.git@pr-draft
 ```
 
-To run the benchmark, clone lm-evaluation-harness, and install it along with some other dependencies
+To run the benchmark, install lm-evaluation-harness and unitxt (www.unitxt.ai), along with some other dependencies
 
 ```bash
-git clone [email protected]:EleutherAI/lm-evaluation-harness.git
-pip install -e path/to/lmeval/.
+pip install lm_eval
+pip install unitxt
 pip install langdetect immutabledict antlr4-python3-runtime==4.11 sacrebleu streamlit boto3 matplotlib loguru
 ```
 
+Link Unitxt to Lm-Eval-Harness (see https://www.unitxt.ai/en/latest/docs/lm_eval.html)
+``` 
+python -c 'from lm_eval.tasks.unitxt import task; import os.path; print("class: !function " + task.__file__.replace("task.py", "task.Unitxt"))' > ./unitxt_cards_for_lm_eval/unitxt
+```
+
 ### Running the benchmark
 
 Running the benchmark can be done using `evaluation/runner.py` that has the following signeture:

evaluation/runner.py

@@ -230,8 +230,8 @@ def run_job(model_id, task_to_run, args):
         ",".join(subtasks_to_run),
         "--output_path",
         output_path,
-        "--cache_requests",
-        "true",
+        # "--cache_requests",
+        # "true",
         "--log_samples",
         "--trust_remote_code",
         # f"--use_cache={cache_dir}",

evaluation/runner_tasks.py

@@ -106,4 +106,30 @@
         {"task": "Winogrande", "num_fewshot": 5, "subtasks": ["winogrande"]},
         {"task": "GSM8k", "num_fewshot": 5, "subtasks": ["gsm8k"]},
     ],
+    "Safety_BBQ": [
+        {"task": "bbq_Age_temp_0_few_shot_5", "num_fewshot": None, "subtasks": ["bbq_Age_temp_0_few_shot_5"]},
+        {"task": "bbq_Disability_status_temp_0_few_shot_5", "num_fewshot": None,
+        "subtasks": ["bbq_Disability_status_temp_0_few_shot_5"]},
+        {"task": "bbq_Gender_identity_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Gender_identity_temp_0_few_shot_5"]},
+        {"task": "bbq_Nationality_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Nationality_temp_0_few_shot_5"]},
+        {"task": "bbq_Physical_appearance_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Physical_appearance_temp_0_few_shot_5"]},
+        {"task": "bbq_Race_ethnicity_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Race_ethnicity_temp_0_few_shot_5"]},
+        {"task": "bbq_Race_x_gender_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Race_x_gender_temp_0_few_shot_5"]},
+        {"task": "bbq_Race_x_SES_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Race_x_SES_temp_0_few_shot_5"]},
+        {"task": "bbq_Religion_temp_0_few_shot_5", "num_fewshot": None, "subtasks": ["bbq_Religion_temp_0_few_shot_5"]},
+        {"task": "bbq_SES_temp_0_few_shot_5", "num_fewshot": None, "subtasks": ["bbq_SES_temp_0_few_shot_5"]},
+        {"task": "bbq_Sexual_orientation_temp_0_few_shot_5", "num_fewshot": None,
+         "subtasks": ["bbq_Sexual_orientation_temp_0_few_shot_5"]},
+    ],
+    "Safety": [
+        {"task": "toxigen", "num_fewshot": 5, "subtasks": ["toxigen"]},
+        {"task": "pop_qa_temp_0_few_shot_5", "num_fewshot": None, "subtasks": ["pop_qa_temp_0_few_shot_5"]},
+        {"task": "crows_pairs_english", "num_fewshot": 5, "subtasks": ["crows_pairs_english"]}
+    ]
 }

evaluation/unitxt_cards_for_lm_eval/bbq_Age_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Age,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Age_temp_0_few_shot_5

evaluation/unitxt_cards_for_lm_eval/bbq_Disability_status_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Disability_status,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Disability_status_temp_0_few_shot_5

evaluation/unitxt_cards_for_lm_eval/bbq_Gender_identity_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Gender_identity,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Gender_identity_temp_0_few_shot_5

evaluation/unitxt_cards_for_lm_eval/bbq_Nationality_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Nationality,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Nationality_temp_0_few_shot_5

evaluation/unitxt_cards_for_lm_eval/bbq_Physical_appearance_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Physical_appearance,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Physical_appearance_temp_0_few_shot_5

evaluation/unitxt_cards_for_lm_eval/bbq_Race_ethnicity_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Race_ethnicity,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Race_ethnicity_temp_0_few_shot_5

evaluation/unitxt_cards_for_lm_eval/bbq_Race_x_SES_temp_0_few_shot_5.yaml

@@ -0,0 +1,3 @@
+include: unitxt
+recipe: card=cards.safety.bbq.Race_x_SES,format=formats.empty,template_card_index=0,demos_pool_size=100,num_demos=5,demos_taken_from=test
+task: bbq_Race_x_SES_temp_0_few_shot_5