From b889c9780150a1b6a7e60b2b3614fef1d1af1f10 Mon Sep 17 00:00:00 2001 From: Yongcheng Yao <49467070+VincentYCYao@users.noreply.github.com> Date: Fri, 27 Jun 2025 11:44:28 +0100 Subject: [PATCH 1/2] bugfix: missing fields in doc when using --log_samples Bug: Fields with "image" in keys or of type dict are not saved in the sample log file. Fix: save dicts and all fields to the JSONL file --- lmms_eval/evaluator.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py index 83c64747e..de460c400 100755 --- a/lmms_eval/evaluator.py +++ b/lmms_eval/evaluator.py @@ -521,13 +521,7 @@ def evaluate( target = task.doc_to_target(doc) saved_doc = {} for key, value in doc.items(): - # If image is not in key - if "image" not in key: - # If audio is also not the value - if isinstance(value, dict) and "array" in value: - continue - else: - saved_doc[key] = value + saved_doc[key] = value filtered_arguments = [] for req in requests: # check if req.args is a list of tuples, and each item in the list is a serializable object From bb53463eabca25f2b2b17607185223a008deb051 Mon Sep 17 00:00:00 2001 From: Bo Li Date: Wed, 23 Jul 2025 13:21:23 +0800 Subject: [PATCH 2/2] lint --- lmms_eval/tasks/camerabench_vqa/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lmms_eval/tasks/camerabench_vqa/utils.py b/lmms_eval/tasks/camerabench_vqa/utils.py index e6ea44681..4244edfe4 100644 --- a/lmms_eval/tasks/camerabench_vqa/utils.py +++ b/lmms_eval/tasks/camerabench_vqa/utils.py @@ -6,7 +6,6 @@ SUFFIX_FOR_VQA = {"yes_no": "Please answer Yes or No.", "multiple_choice": "Please output the letter corresponding to the correct option."} - def get_scores(scores): """ Calculate various scores based on the given results. @@ -148,11 +147,8 @@ def find_word_position(string, word): def cambench_doc_to_visual(doc): try: - default_path = os.path.join(os.getenv('HOME'), '.cache/huggingface') - load_path = os.path.expanduser(os.path.join( - os.getenv("HF_HOME", default_path), - 'camerabench_vqa/datasets--chancharikm--camerabench_vqa_lmms_eval/snapshots' - )) + default_path = os.path.join(os.getenv("HOME"), ".cache/huggingface") + load_path = os.path.expanduser(os.path.join(os.getenv("HF_HOME", default_path), "camerabench_vqa/datasets--chancharikm--camerabench_vqa_lmms_eval/snapshots")) if not os.path.exists(load_path): raise FileNotFoundError(f"Dataset path not found: {load_path}")