Add jdocqa overall score and validate metrics argument

Author: speed1313

examples/sample.py

@@ -31,9 +31,27 @@
     "--metrics",
     type=str,
     default="llm_as_a_judge_heron_bench",
-    help="metrics to evaluate. You can specify multiple metrics separated by comma (e.g. --metrics exact_match,rougel).",
+    help="metrics to evaluate. You can specify multiple metrics separated by comma (e.g. --metrics exact_match,llm_as_a_judge) You can use rougel,substring_match,jmmmu,jdocqa,llm_as_a_judge_heron_bench,exact_match",
 )
 
+valid_metrics = [
+    "rougel",
+    "substring_match",
+    "jmmmu",
+    "jdocqa",
+    "llm_as_a_judge_heron_bench",
+    "exact_match",
+]
+
+
+def validate_metrics(metrics: list[str]):
+    for metric in metrics:
+        if metric not in valid_metrics:
+            raise ValueError(
+                f"Invalid metric: {metric}. Valid metrics are {valid_metrics}"
+            )
+
+
 args = parser.parse_args()
 
 gen_kwargs = GenerationConfig(
@@ -105,6 +123,7 @@
 # evaluate the predictions
 
 metrics = args.metrics.split(",")
+validate_metrics(metrics)
 
 scores_for_each_metric = {}
 

src/eval_mm/metrics/jdocqa_scorer.py

@@ -25,6 +25,7 @@ def jdocqa_normalize(text):
 
 
 def bleu_ja(refs, pred):
+    """Calculate BLEU score for Japanese text. Score is normalized to [0, 1]."""
     bleu_score = sentence_bleu(
         hypothesis=pred,
         references=refs,
@@ -34,7 +35,7 @@ def bleu_ja(refs, pred):
         use_effective_order=False,
         lowercase=False,
     )
-    return bleu_score.score
+    return bleu_score.score / 100
 
 
 class JDocQAScorer(Scorer):
@@ -83,6 +84,7 @@ def aggregate(scores: list[int], **kwargs) -> dict:
                 metrics[key] = 0
                 continue
             metrics[key] = sum(value) / len(value)
+        metrics["overall"] = sum(scores) / len(scores)
 
         return metrics