[FIX] Resolve MMMU-test submission file generation issue (#724)

* fix(mmmu): resolve MMMU-test submission file generation issue

* fix(mmmu): Increase max_new_tokens to 512 for Qwen2.5-VL series models

* style(*): Fix linting issues

Author: xyyandxyy

lmms_eval/tasks/mmmu/mmmu_test.yaml

@@ -8,6 +8,11 @@ doc_to_target: "answer"
 # The return value of process_results will be used by metrics
 process_results: !function utils.mmmu_process_results
 
+generation_kwargs: 
+# Set max_new_tokens to a higher value (e.g., 512) for models like Qwen2.5-VL series
+# that tend to generate longer, more detailed responses and require larger token limits
+  max_new_tokens: 512
+
 metric_list:
   - metric: submission
     aggregation: !function utils.mmmu_test_aggregate_results_for_submission

lmms_eval/tasks/mmmu/utils.py

@@ -159,11 +159,11 @@ def mmmu_process_results(doc, results):
             parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans)
         else:
             parsed_pred = parse_open_response(pred)
-
+            parsed_pred = str(parsed_pred[0]) if parsed_pred else ""
         parsed_preds.append(parsed_pred)
-
+    mmmu_submission = {doc["id"]: parsed_preds[0]}
     mmmu_exact_acc = {"id": doc["id"], "subdomain": extract_subset_name(doc["id"]), "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": parsed_preds}
-    return {"mmmu_acc": mmmu_exact_acc, "mmmu_acc_pass_at_k": mmmu_exact_acc}
+    return {"mmmu_acc": mmmu_exact_acc, "mmmu_acc_pass_at_k": mmmu_exact_acc, "submission": mmmu_submission}
 
 
 def mmmu_reasoning_process_results(doc, results):

lmms_eval/tasks/mmsi_bench/utils.py

@@ -1,13 +1,14 @@
+import io
 import logging
 import re
-from PIL import Image
+from collections import defaultdict
+
 import numpy as np
-import io
 import pandas as pd
-from collections import defaultdict
+from PIL import Image
+
 from lmms_eval.filters.extraction import ExtendedRegexFilter
 from lmms_eval.filters.transformation import MapFilter
-import re
 
 eval_logger = logging.getLogger("lmms-eval")
 
@@ -31,30 +32,28 @@ def msr_doc_to_visual(doc):
     return image_list
 
 
-
-
 def extract_single_choice_with_word_boundary(pred, gt):
-    pattern_1 = r'``([^`]*)``'
+    pattern_1 = r"``([^`]*)``"
     match = re.search(pattern_1, pred)
     if match:
-        pred = match.group(1)  
+        pred = match.group(1)
 
-    pattern_2 = r'`([^`]*)`'
+    pattern_2 = r"`([^`]*)`"
     match = re.search(pattern_2, pred)
     if match:
-        pred = match.group(1)  
+        pred = match.group(1)
 
-    pattern_add = r'\{([^}]*)\}'
+    pattern_add = r"\{([^}]*)\}"
     match = re.search(pattern_add, pred)
     if match:
-        pred = match.group(1)  
+        pred = match.group(1)
 
-    pattern_3 = r'\b[A-D]\b(?!\s[a-zA-Z])'
+    pattern_3 = r"\b[A-D]\b(?!\s[a-zA-Z])"
     match = re.search(pattern_3, pred)
     if match:
-        pred = match.group()  
+        pred = match.group()
     else:
-        return None 
+        return None
 
     answer = gt.lower().replace("\n", " ").strip()
     predict = pred.lower().replace("\n", " ").strip()
@@ -72,7 +71,6 @@ def extract_single_choice_with_word_boundary(pred, gt):
     return 0.0
 
 
-
 def msr_process_results(doc, results):
     """
     Args:
@@ -114,4 +112,3 @@ def msr_aggregate_results(results):
     all_scores = [score for scores in l2_category_scores.values() for score in scores]
     avg_score = sum(all_scores) / len(all_scores) if all_scores else 0.0
     return avg_score
-