[fix] cli_evaluate to properly handle Namespace arguments (#733)

* Fix cli_evaluate to properly handle Namespace arguments

This fixes an issue where cli_evaluate would fail when called programmatically
with an argparse.Namespace object and empty sys.argv. The function now:

1. Always parses default arguments first
2. Only checks sys.argv when no args are provided (args is None)
3. Merges provided Namespace attributes with defaults

This enables the function to be called from other Python modules while
maintaining backward compatibility with command-line usage.

Fixes the issue where passing args as Namespace fails due to sys.argv check.

* lint

Author: Luodian

lmms_eval/__main__.py

@@ -271,18 +271,23 @@ def parse_eval_args() -> argparse.Namespace:
 
 
 def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
-    if not args:
-        args = parse_eval_args()
+    default_args = parse_eval_args()
 
-    # Check if no arguments were passed after parsing
-    if len(sys.argv) == 1:
+    if args is None and len(sys.argv) == 1:
         print("┌───────────────────────────────────────────────────────────────────────────────┐")
         print("│ Please provide arguments to evaluate the model. e.g.                          │")
         print("│ `lmms-eval --model llava --model_path liuhaotian/llava-v1.6-7b --tasks okvqa` │")
         print("│ Use `lmms-eval --help` for more information.                                  │")
         print("└───────────────────────────────────────────────────────────────────────────────┘")
         sys.exit(1)
 
+    # If args were provided, override the defaults
+    if args:
+        for key, value in vars(args).items():
+            setattr(default_args, key, value)
+
+    args = default_args
+
     if args.wandb_args:
         if "name" not in args.wandb_args:
             name = f"{args.model}_{args.model_args}_{utils.get_datetime_str(timezone=args.timezone)}"

lmms_eval/tasks/camerabench_vqa/utils.py

@@ -6,7 +6,6 @@
 SUFFIX_FOR_VQA = {"yes_no": "Please answer Yes or No.", "multiple_choice": "Please output the letter corresponding to the correct option."}
 
 
-
 def get_scores(scores):
     """
     Calculate various scores based on the given results.
@@ -148,11 +147,8 @@ def find_word_position(string, word):
 
 def cambench_doc_to_visual(doc):
     try:
-        default_path = os.path.join(os.getenv('HOME'), '.cache/huggingface')
-        load_path = os.path.expanduser(os.path.join(
-            os.getenv("HF_HOME", default_path),
-            'camerabench_vqa/datasets--chancharikm--camerabench_vqa_lmms_eval/snapshots'
-        ))
+        default_path = os.path.join(os.getenv("HOME"), ".cache/huggingface")
+        load_path = os.path.expanduser(os.path.join(os.getenv("HF_HOME", default_path), "camerabench_vqa/datasets--chancharikm--camerabench_vqa_lmms_eval/snapshots"))
 
         if not os.path.exists(load_path):
             raise FileNotFoundError(f"Dataset path not found: {load_path}")