Fix Heron-bench scoring and Add Asagi model (#146)

* Fix Heron-bench scoring

* Add Asagi

* Emphasize top1 score

Author: speed1313

README.md

@@ -76,8 +76,14 @@ uv run --group normal python examples/sample.py \
   --overwrite
 ```
 
-The evaluation score and output results will be saved in
-`test/{task_id}/{model_id}/evaluation.jsonl` and `test/{task_id}/{model_id}/prediction.jsonl`.
+The evaluation score and model outputs will be saved in the `result` directory like below:
+```
+├── japanese-heron-bench
+│   ├── llava-hf
+│   │   ├── llava-1.5-7b-hf
+│   │   │   ├── evaluation.jsonl
+│   │   │   └── prediction.jsonl
+```
 
 If you want to evaluate multiple models on multiple tasks, please check `eval_all.sh`.
 

examples/Asagi.py

@@ -0,0 +1,53 @@
+import torch
+from PIL import Image
+from transformers import AutoModel, AutoProcessor
+from base_vlm import BaseVLM
+from utils import GenerationConfig
+
+
+class VLM(BaseVLM):
+    def __init__(self, model_id: str = "MIL-UT/Asagi-14B") -> None:
+        self.model_id = model_id
+        self.model = AutoModel.from_pretrained(
+            self.model_id,
+            trust_remote_code=True,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+        )
+        self.processor = AutoProcessor.from_pretrained(self.model_id)
+
+    def generate(
+        self,
+        images: list[Image.Image],
+        text: str,
+        gen_kwargs: GenerationConfig = GenerationConfig(),
+    ) -> str:
+        prompt = f"""以下は、タスクを説明する指示です。要求を適切に満たす応答を書きなさい。
+        ### 指示:
+        {"<image>"*len(images)}
+        {text}
+        ### 応答:
+        """
+
+        if len(images) == 0:
+            images = None
+        inputs = self.processor(text=prompt, images=images, return_tensors="pt")
+        inputs_text = self.processor.tokenizer(prompt, return_tensors="pt")
+        inputs["input_ids"] = inputs_text["input_ids"]
+        inputs["attention_mask"] = inputs_text["attention_mask"]
+        inputs = {
+            k: inputs[k].to(self.model.device) for k in inputs if k != "token_type_ids"
+        }
+
+        generate_ids = self.model.generate(**inputs, **gen_kwargs.__dict__)
+        generated_text = self.processor.batch_decode(
+            generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )[0]
+        # truncate the text to remove the prompt
+        generated_text = generated_text.split("### 応答:")[1].strip()
+        return generated_text
+
+
+if __name__ == "__main__":
+    vlm = VLM()
+    vlm.test_vlm()

examples/base_vlm.py

@@ -19,12 +19,6 @@ def generate(
 
     def test_vlm(self):
         """Test the model with one or two images."""
-        output = self.generate([], "画像には何が映っていますか?")
-        logger.info(f"Output: {output}")
-        assert isinstance(
-            output, str
-        ), f"Expected output to be a string, but got {type(output)}"
-
         image_file = "http://images.cocodataset.org/val2017/000000039769.jpg"
         image = Image.open(requests.get(image_file, stream=True).raw)
         image_file2 = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg"
@@ -40,3 +34,9 @@ def test_vlm(self):
         assert isinstance(
             output, str
         ), f"Expected output to be a string, but got {type(output)}"
+
+        output = self.generate([], "画像には何が映っていますか?")
+        logger.info(f"Output: {output}")
+        assert isinstance(
+            output, str
+        ), f"Expected output to be a string, but got {type(output)}"

examples/japanese_stable_vlm.py

@@ -181,16 +181,15 @@ def generate(
         # instruct blip does not expect the <image> tag
         prompt = build_prompt(task="vqa", input=text)
         if len(images) == 0:
-            images = None
-            inputs = self.processor(text=prompt, return_tensors="pt")
+            raise ValueError("Please provide at least one image.")
+
         else:
             images = [process_images(images)]
             inputs = self.processor(images=images, return_tensors="pt", truncation=True)
             text_encoding = self.tokenizer(
                 prompt, add_special_tokens=False, return_tensors="pt"
             )
             inputs.update(text_encoding)
-
         # autoregressively complete prompt
         output = self.model.generate(
             **inputs.to(self.device, dtype=self.model.dtype), **gen_kwargs.__dict__

examples/model_table.py

@@ -35,6 +35,7 @@
     "sbintuitions/sarashina2-vision-8b": "sarashina.VLM",
     "sbintuitions/sarashina2-vision-14b": "sarashina.VLM",
     "microsoft/Phi-4-multimodal-instruct": "phi_4_mm.VLM",
+    "MIL-UT/Asagi-14B": "Asagi.VLM",
 }
 
 

examples/sample.py

@@ -15,7 +15,7 @@ def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--model_id", type=str, default="llava-hf/llava-1.5-7b-hf")
     parser.add_argument("--task_id", type=str, default="japanese-heron-bench")
-    parser.add_argument("--judge_model", type=str, default="gpt-4o-mini-2024-07-18")
+    parser.add_argument("--judge_model", type=str, default="gpt-4o-2024-11-20")
     parser.add_argument("--batch_size_for_evaluation", type=int, default=10)
     parser.add_argument("--overwrite", action="store_true")
     parser.add_argument("--result_dir", type=str, default="result")
@@ -85,6 +85,11 @@ def parse_args():
         logger.info(task.dataset)
         error_count = 0
         for doc in tqdm(task.dataset):
+            if error_count > len(task.dataset) * 0.1:
+                logger.error(
+                    f"Error count is too high. Error count: {error_count}, Dataset length: {len(task.dataset)}. You need to re-run the evaluation."
+                )
+                exit()
             images = task.doc_to_visual(doc)
             text = task.doc_to_text(doc)
             if "<image>" in text:

scripts/browse_prediction.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from datasets import load_dataset
 import random
 import eval_mm
 from argparse import ArgumentParser

scripts/make_leaderboard.py

@@ -2,7 +2,7 @@
 import os
 import pandas as pd
 from argparse import ArgumentParser
-from typing import Dict, List, Optional
+from typing import List, Optional
 from loguru import logger
 import eval_mm
 import eval_mm.metrics
@@ -35,7 +35,12 @@
 }
 
 
-def main(result_dir: str, model_list: List[str], output_path: Optional[str] = None):
+def main(
+    result_dir: str,
+    model_list: List[str],
+    output_path: Optional[str] = None,
+    output_format: str = "markdown",
+):
     task_dirs = [d for d in os.listdir(result_dir) if not d.startswith(".")]
 
     df = pd.DataFrame()
@@ -67,6 +72,8 @@ def main(result_dir: str, model_list: List[str], output_path: Optional[str] = No
         df = df._append(model_results, ignore_index=True)
 
     df = df.set_index("Model")
+    # round to 2 decimal places
+    df = df.round(2)
     df = df.rename(
         columns={
             k: f"{TASK_ALIAS[k.split('/')[0]]}/{METRIC_ALIAS[k.split('/')[1]]}"
@@ -76,16 +83,31 @@ def main(result_dir: str, model_list: List[str], output_path: Optional[str] = No
     # sort columns
     df = df.reindex(sorted(df.columns), axis=1)
 
-    print(df.to_markdown(mode="github"))
+    # textbf top1 score for each column
+    for col in df.columns:
+        top1_model = df[col].idxmax()
+        if output_format == "latex":
+            df.loc[top1_model, col] = f"\\textbf{{{df.loc[top1_model, col]}}}"
+        else:
+            df.loc[top1_model, col] = f"**{df.loc[top1_model, col]}**"
+
+    if output_format == "markdown":
+        table = df.to_markdown(mode="github", floatfmt=".2f")
+    elif output_format == "latex":
+        table = df.to_latex(float_format="%.2f")
+    print(table)
 
     with open(output_path, "w") as f:
-        f.write(df.to_markdown(mode="github"))
+        f.write(table)
 
 
 def parse_args():
     parser = ArgumentParser()
     parser.add_argument("--result_dir", type=str, default="result")
     parser.add_argument("--output_path", type=str, default="leaderboard.md")
+    parser.add_argument(
+        "--output_format", type=str, default="markdown", choices=["markdown", "latex"]
+    )
     return parser.parse_args()
 
 
@@ -94,11 +116,29 @@ def parse_args():
 
     # モデルは実行時引数でも取れるようにしても良い
     model_list = [
-        "Qwen/Qwen2.5-VL-7B-Instruct",
+        "stabilityai/japanese-instructblip-alpha",
+        "stabilityai/japanese-stable-vlm",
+        "SakanaAI/Llama-3-EvoVLM-JP-v2",
+        "cyberagent/llava-calm2-siglip",
+        "llm-jp/llm-jp-3-vila-14b",
         "sbintuitions/sarashina2-vision-8b",
         "sbintuitions/sarashina2-vision-14b",
-        "google/gemma-3-12b-it",
+        "MIL-UT/Asagi-14B",
         "llava-hf/llava-1.5-7b-hf",
+        "llava-hf/llava-v1.6-mistral-7b-hf",
+        "neulab/Pangea-7B-hf",
+        "mistralai/Pixtral-12B-2409",
+        "meta-llama/Llama-3.2-11B-Vision-Instruct",
+        "Efficient-Large-Model/VILA1.5-13b",
+        "OpenGVLab/InternVL2-8B",
+        "OpenGVLab/InternVL2-26B",
+        "Qwen/Qwen2.5-VL-7B-Instruct",
+        "Qwen/Qwen2.5-VL-72B-Instruct",
+        "google/gemma-3-4b-it",
+        "google/gemma-3-12b-it",
+        "google/gemma-3-27b-it",
+        "microsoft/Phi-4-multimodal-instruct",
+        "gpt-4o-2024-11-20",
     ]
 
-    main(args.result_dir, model_list, args.output_path)
+    main(args.result_dir, model_list, args.output_path, args.output_format)