CC_OCR 日本語サブセットの追加 (#168)

* add CCOCR Task

* add cvqa japanese subset

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactoring and add results

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: speed1313 <speedtry13@icloud.com>

Author: 3 people

eval_all.sh

@@ -30,6 +30,7 @@ declare -A MODEL_GROUP_MAP=(
     ["sbintuitions/sarashina2-vision-8b"]="sarashina"
     ["sbintuitions/sarashina2-vision-14b"]="sarashina"
     ["microsoft/Phi-4-multimodal-instruct"]="phi"
+    ["turing-motors/Heron-NVILA-Lite-15B"]="heron_nvila"
 )
 
 # Task list
@@ -44,6 +45,8 @@ declare -a task_list=(
     "llava-bench-in-the-wild"
     "jic-vqa"
     "mecha-ja"
+    "cc-ocr"
+    "cvqa"
 )
 
 # Define metrics per task
@@ -58,6 +61,8 @@ declare -A METRIC_MAP=(
     ["llava-bench-in-the-wild"]="llm-as-a-judge,rougel"
     ["jic-vqa"]="jic-vqa"
     ["mecha-ja"]="mecha-ja"
+    ["cc-ocr"]="cc-ocr"
+    ["cvqa"]="substring-match"
 )
 
 # Result directories
@@ -77,9 +82,7 @@ for RESULT_DIR in "${result_dir_list[@]}"; do
                 --task_id "$task" \
                 --metrics "$METRIC" \
                 --judge_model "gpt-4o-2024-11-20" \
-                --result_dir "$RESULT_DIR" \
-                --inference_only
-
+                --result_dir "$RESULT_DIR"
         done
     done
 done

examples/heron_nvila.py

@@ -37,7 +37,9 @@ def __init__(self, model_id="turing-motors/Heron-NVILA-Lite-15B"):
             model_id, trust_remote_code=True, device_map="auto"
         )
 
-    def generate(self, image, text: str, gen_kwargs: GenerationConfig = GenerationConfig()):
+    def generate(
+        self, image, text: str, gen_kwargs: GenerationConfig = GenerationConfig()
+    ):
         gen_cfg = HFGenerationConfig(**gen_kwargs.__dict__)
         prompt = create_prompt(text, image)
         with torch.no_grad():

github_pages/public/dataset_url.json

@@ -28,5 +28,11 @@
     },
     "MECHA": {
         "url": "https://huggingface.co/datasets/llm-jp/MECHA-ja"
+    },
+    "CC-OCR": {
+        "url": "https://huggingface.co/datasets/wulipc/CC-OCR"
+    },
+    "CVQA": {
+        "url": "https://huggingface.co/datasets/afaji/cvqa"
     }
 }