Support CLI-based Full Evaluation

Author: w11wo

evaluation/classification/eval_classification.py

@@ -1,6 +1,8 @@
 # Modified from: https://github.com/mrpeerat/Thai-Sentence-Vector-Benchmark/blob/main/Transfer_Evaluation/transfer.py
 
 from dataclasses import dataclass
+import json
+import os
 
 from datargs import parse
 from datasets import load_dataset
@@ -19,9 +21,12 @@ class Args:
     text_column: str = "tweet"
     label_column: str = "label"
     encode_batch_size: int = 128
+    output_folder: str = "results"
 
 
 def main(args: Args):
+    os.makedirs(args.output_folder, exist_ok=True)
+
     model = SentenceTransformer(args.model_name)
 
     dataset = load_dataset(args.dataset_name, args.dataset_config)
@@ -46,9 +51,7 @@ def main(args: Args):
     predictions = classifier.predict(test_text_encoded)
 
     acc = accuracy_score(test_ds[args.label_column], predictions)
-    precision, recall, f1, _ = precision_recall_fscore_support(
-        test_ds[args.label_column], predictions, average="macro"
-    )
+    precision, recall, f1, _ = precision_recall_fscore_support(test_ds[args.label_column], predictions, average="macro")
 
     results = {
         "accuracy": acc,
@@ -57,7 +60,9 @@ def main(args: Args):
         "f1": f1,
     }
 
-    print(results)
+    task_name = f"{args.dataset_name.split('/')[-1]}_{args.dataset_config}"
+    with open(f"{args.output_folder}/{task_name}_{args.test_split_name}.json", "w") as f:
+        json.dump(results, f, indent=4)
 
 
 if __name__ == "__main__":

evaluation/pair_classification/eval_pair_classification.py

@@ -1,6 +1,8 @@
 # Modified from: https://github.com/embeddings-benchmark/mteb/blob/main/mteb/evaluation/evaluators/PairClassificationEvaluator.py
 
 from dataclasses import dataclass
+import json
+import os
 
 import numpy as np
 from datargs import parse
@@ -22,6 +24,7 @@ class Args:
     neutral_label: int = 1
     contradiction_label: int = 2
     encode_batch_size: int = 128
+    output_folder: str = "results"
 
 
 def compute_metrics(model, sentences_1, sentences_2, labels, batch_size):
@@ -136,6 +139,8 @@ def ap_score(scores, labels, high_score_more_similar: bool):
 
 
 def main(args: Args):
+    os.makedirs(args.output_folder, exist_ok=True)
+
     model = SentenceTransformer(args.model_name)
 
     test_ds = load_dataset(args.dataset_name, split=args.test_split_name, trust_remote_code=True)
@@ -157,7 +162,8 @@ def main(args: Args):
     main_score = max(scores[short_name]["ap"] for short_name in scores)
     scores["main_score"] = main_score
 
-    print(scores)
+    with open(f"{args.output_folder}/{args.dataset_name}_{args.test_split_name}.json", "w") as f:
+        json.dump(scores, f, indent=4)
 
 
 if __name__ == "__main__":

evaluation/run_evaluation.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+model=$1
+model_name="${model#*/}"
+
+###############################
+# RETRIEVAL
+###############################
+
+python retrieval/eval_tydiqa.py \
+    --model-name $model \
+    --test-dataset-name khalidalt/tydiqa-goldp \
+    --test-dataset-config indonesian \
+    --test-dataset-split validation \
+    --test-batch-size 32 \
+    --output-folder retrieval/results/$model_name
+
+python retrieval/eval_miracl.py \
+    --model-name $model \
+    --test-dataset-name miracl/miracl \
+    --test-dataset-config id \
+    --test-dataset-split dev \
+    --test-batch-size 32 \
+    --output-folder retrieval/results/$model_name
+
+###############################
+# PAIR CLASSIFICATION
+###############################
+
+for split in test_lay test_expert
+do
+  python pair_classification/eval_pair_classification.py \
+      --model-name $model \
+      --dataset-name indonli \
+      --test-split-name $split \
+      --text-column-1 premise \
+      --text-column-2 hypothesis \
+      --label-column label \
+      --output-folder pair_classification/results/$model_name
+done
+
+###############################
+# CLASSIFICATION
+###############################
+
+python classification/eval_classification.py \
+    --model-name $model \
+    --dataset-name indonlp/indonlu \
+    --dataset-config emot \
+    --train-split-name train \
+    --test-split-name test \
+    --text-column tweet \
+    --label-column label \
+    --output-folder classification/results/$model_name
+
+python classification/eval_classification.py \
+    --model-name $model \
+    --dataset-name indonlp/indonlu \
+    --dataset-config smsa \
+    --train-split-name train \
+    --test-split-name test \
+    --text-column text \
+    --label-column label \
+    --output-folder classification/results/$model_name
+
+mteb \
+    -m $model \
+    -l id \
+    --output_folder mteb/results/$model_name
+
+###############################
+# SEMANTIC TEXTUAL SIMILARITY
+###############################
+
+python sts/eval_sts.py \
+    --model-name $model \
+    --test-dataset-name LazarusNLP/stsb_mt_id \
+    --test-dataset-split test \
+    --test-text-column-1 text_1 \
+    --test-text-column-2 text_2 \
+    --test-label-column correlation \
+    --test-batch-size 32 \
+    --output-folder sts/results/$model_name

evaluation/sts/eval_sts.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+import os
 
 from datargs import parse
 from datasets import load_dataset
@@ -15,9 +16,12 @@ class Args:
     test_text_column_2: str = "text_2"
     test_label_column: str = "correlation"
     test_batch_size: int = 32
+    output_folder: str = "results"
 
 
 def main(args: Args):
+    os.makedirs(args.output_folder, exist_ok=True)
+
     model = SentenceTransformer(args.model_name)
 
     # Load dataset
@@ -31,11 +35,8 @@ def main(args: Args):
         for data in test_ds
     ]
 
-    evaluator = EmbeddingSimilarityEvaluator.from_input_examples(
-        test_data, batch_size=args.test_batch_size
-    )
-
-    print(evaluator(model))
+    evaluator = EmbeddingSimilarityEvaluator.from_input_examples(test_data, batch_size=args.test_batch_size)
+    evaluator(model, output_path=args.output_folder)
 
 
 if __name__ == "__main__":