fixed a bug that invalidates base

Author: Haozhe Qi

llava/action/ek_eval.py

@@ -342,7 +342,6 @@ def collate_fn(batch):
     if eval_args.learn_neighbor_actions:
         from llava.action.generate_interval_pred import  get_lookup_dict
         if eval_args.test_type.startswith('temporal_cot'):
-
             lookup_table = get_lookup_dict(eval_args.val_metadata, test_type = eval_args.test_type, pseudo_folder = eval_args.pseudo_folder)
 
 
@@ -353,7 +352,7 @@ def collate_fn(batch):
             mc_data = mc_data[0]
             time_meta = time_meta[0]
 
-            if eval_args.learn_neighbor_actions:
+            if eval_args.learn_neighbor_actions and lookup_table:
                 _id = time_meta['vid_path']
                 _id = _id.replace('/', '-')
                 uid = f"{_id}_{time_meta['start_second']}_{time_meta['end_second']}"

llava/action/generate_interval_pred.py

@@ -172,7 +172,13 @@ def get_lookup_dict(ann_file, test_type = 'base', delta = 3, pseudo_folder = Non
                                'prev2_offset': round(start_times[i+2] - start_times[i],2),
                                 'prev1_narration': narration2,
                                 'prev1_offset': round(start_times[i+2] - start_times[i+1],2),
-                                'cur_narration': narration3}
+                                'cur_narration': narration3,
+                                'prev2_start': start_times[i],
+                                'prev2_end': end_times[i],
+                                'prev1_start': start_times[i+1],
+                                'prev1_end': end_times[i+1],
+                                'cur_start': start_times[i+2],
+                                'cur_end': end_times[i+2]}
     return table
 
 

llava/action/utils.py

@@ -254,7 +254,11 @@ def format_task_related_prompt(question, question_type, meta_data = None, perspe
             assert isinstance(question, list)
             suffix = ", ".join(question)           
             suffix = f"{prev2_offset} seconds ago, you started an action {prev2_narration}. {prev1_offset} seconds ago, you started an action {prev1_narration}. What action are you currently performing? Here are the options of actions you can select:\n" + suffix 
-            ret = prefix + suffix         
+            ret = prefix + suffix 
+            rank0_print('debug')
+            rank0_print (meta_data)
+            rank0_print (ret)
+            
         else:
             action_rep_suffix = "Given multiple choices, format your answer briefly such as 'A. move knife'. "              
             prefix = f"{perspective_prefix}{action_rep_suffix}\n"