debug

Author: Haozhe Qi

llava/action/ek_eval.py

@@ -128,6 +128,7 @@ def get_args_parser():
     parser.add_argument('--learn_neighbor_actions', action='store_true', default = False)
     parser.add_argument('--pseudo_folder', default = None, type = str)
     parser.add_argument('--output_dir', default = None, type = str)
+    parser.add_argument("--perspective", default = "first_person", type = str)
     return parser
 
 def prepare_llava(pretrained):
@@ -169,6 +170,7 @@ def ensemble_llava_evaluation(
                               learn_neighbor_actions = False,                             
                               time_meta = None,
                               meta_data = None,
+                              perspective = "first_person"
                               ):
     """
     This function tests how consistent the model is if we shuffle the position of the answers
@@ -206,7 +208,8 @@ def ensemble_llava_evaluation(
                             temperature = temperature,
                             time_meta = time_meta,
                             learn_neighbor_actions = learn_neighbor_actions,
-                            meta_data = meta_data
+                            meta_data = meta_data,
+                            perspective = perspective
                             )
         # remove the trailing comma if there is one
         pred = pred.rstrip(',')
@@ -386,7 +389,9 @@ def collate_fn(batch):
                                                         test_type = eval_args.test_type,  
                                                         learn_neighbor_actions = eval_args.learn_neighbor_actions,                                                    
                                                         time_meta = time_meta,
-                                                        meta_data = meta_data)
+                                                        meta_data = meta_data,
+                                                        perspective = eval_args.perspective
+                                                        )
 
 
 

llava/action/llava_inference.py

@@ -21,7 +21,8 @@ def llava_inference(
     test_type = 'base',
     time_meta = None,
     learn_neighbor_actions = False,
-    meta_data = None
+    meta_data = None,
+    perspective = "first_person"
     ):
 
         model.eval()              
@@ -74,6 +75,7 @@ def llava_inference(
                                         "mc_top5_official_key",
                                         include_frame_time = False,
                                         learn_neighbor_actions = learn_neighbor_actions,
+                                        perspective = perspective,
                                         include_time_instruction= False)
 
             question = f"You observed the video before and wrote down the notes: {caption_answer}. Now you watch the same video again and you can do better. " +  question                             
@@ -87,6 +89,7 @@ def llava_inference(
                                         include_frame_time = False,
                                         learn_neighbor_actions = learn_neighbor_actions,
                                         include_time_instruction= False,
+                                        perspective = perspective,
                                         meta_data=meta_data)
 
 

llava/action/utils.py

@@ -328,13 +328,17 @@ def format_llava_prompt(image_token,
                         include_time_instruction = False,
                         include_frame_time = False,
                         meta_data = None,
-                        learn_neighbor_actions = False
+                        learn_neighbor_actions = False,
+                        perspective = "first_person"
                         ):
     """
     baseline llava prompt: {image_token}\n{task_related_prompt}
     with time instruction: {image_token}\n{time_instruction}\n{task_related_prompt}
     """
-    task_related_prompt = format_task_related_prompt(question, question_type, meta_data = meta_data, learn_neighbor_actions = learn_neighbor_actions)
+    task_related_prompt = format_task_related_prompt(question, question_type, 
+                                                     meta_data = meta_data, 
+                                                     learn_neighbor_actions = learn_neighbor_actions,
+                                                     perspective = perspective)
 
     time_instruction =  format_time_instruction(video_duration, n_frames, include_frame_time)
 

llava/train/llava_trainer.py

@@ -496,6 +496,10 @@ def __init__(self,
         self.model_max_length = model_max_length
 
     def evaluate(self, eval_dataset=None, ignore_keys=None, metric_key_prefix="eval", eval_result_folder = None):                
+        
+        print ('debug')
+        print (self.eval_args)
+        
         accuracy = evaluate_on_EK100(self.eval_args, self.model, self.tokenizer, eval_result_folder = eval_result_folder)
         metrics = {f"{metric_key_prefix}_EK100_accuracy": accuracy}
         self.log(metrics)

llava/train/train.py

@@ -181,23 +181,6 @@ class TrainingArguments(transformers.TrainingArguments):
     attn_implementation: str = field(default='flash_attention_2', metadata={"help": "Use transformers attention implementation."})
     overwrite_output_dir: bool =True
 
-# @dataclass
-# class EvaluationArguments:
-#     eval_num_processes: int = field(default=1)
-#     task_names: str = field(default=None)
-#     model: str = field(default="llava")
-#     model_args: Optional[str] = field(default=None)
-#     num_fewshot: Optional[int] = field(default=None)
-#     batch_size: int = field(default=1)
-#     device: Optional[str] = field(default=None)
-#     limit: Optional[int] = field(default=None)
-#     check_integrity: Optional[bool] = field(default=False)
-#     show_task_to_terminal: Optional[bool] = field(default=False)
-#     log_samples: Optional[bool] = field(default=True)
-#     gen_kwargs: Optional[str] = field(default="")
-#     log_samples_suffix: Optional[str] = field(default="")
-#     output_path: Optional[str] = field(default="./logs/")
-
 # for EK100
 @dataclass
 class EK100EvalArguments:
@@ -219,6 +202,7 @@ class EK100EvalArguments:
     n_narrations: int = -1
     test_type: str = 'base'
     learn_neighbor_actions: bool = False
+    perspective: str = "first_person"
 
 def maybe_zero_3(param, ignore_status=False, name=None):
     from deepspeed import zero
@@ -1327,7 +1311,8 @@ def _get_item(self, i) -> Dict[str, torch.Tensor]:
                                                  include_time_instruction= self.data_args.add_time_instruction,
                                                  meta_data = meta_data,                                                 
                                                  include_frame_time = False,
-                                                 learn_neighbor_actions = self.eval_args.learn_neighbor_actions)
+                                                 learn_neighbor_actions = self.eval_args.learn_neighbor_actions,
+                                                 perspective = self.eval_args.perspective)
                     sources[0]["conversations"][0]["value"] = llava_prompt
                     # rank0_print (sources[0])
 

run_llmseval_clariden.sbatch

@@ -50,19 +50,11 @@ LAUNCHER="accelerate launch \
     "
 
 PYTHON_FILE="-m lmms_eval"
-# PYTHON_ARGS=" \
-#     --model llava_onevision \
-#     --model_args pretrained=experiments/llava-onevision-qwen2-0.5b-ov,conv_template=qwen_1_5,model_name=llava_qwen \
-#     --tasks video_dc499 \
-#     --batch_size 1 \
-#     --log_samples_suffix llava_onevision \
-#     --output_path ./logs/ \
-#     --verbosity=DEBUG \
-#     "
+
 
 PYTHON_ARGS=" \
     --model llava_vid \
-    --model_args pretrained=lmms-lab/LLaVA-Video-7B-Qwen2,conv_template=qwen_1_5,max_frames_num=64,mm_spatial_pool_mode=average \
+    --model_args pretrained=experiments/LLaVA-Video-7B-Qwen2,conv_template=qwen_1_5,max_frames_num=64,mm_spatial_pool_mode=average \
     --tasks videomme \
     --batch_size 1 \
     --log_samples \