Merge branch 'shaokai/dev' of github.com:yeshaokai/LLaVA-NeXT into shaokai/dev

Author: Ye Shaokai

llava/action/calculate_action_model_acc.py

@@ -130,6 +130,7 @@ def get_args_parser():
     parser.add_argument('--output_dir', default = None, type = str)
     parser.add_argument("--perspective", default = "first_person", type = str)
     parser.add_argument('--benchmark_testing', action='store_true', default = False)
+    parser.add_argument('--include_time_instruction', action='store_true', default = False)
     return parser
 
 def prepare_llava(pretrained):
@@ -345,7 +346,7 @@ def collate_fn(batch):
         from llava.action.generate_interval_pred import  get_lookup_dict
         if eval_args.test_type.startswith('temporal_cot'):
             lookup_table = get_lookup_dict(eval_args.val_metadata, 
-                                           eval_args.action_representation,
+                                           'GT_random_narration',
                                            test_type = eval_args.test_type, 
                                            pseudo_folder = eval_args.pseudo_folder)
 

llava/action/ek_eval.py

@@ -268,12 +268,8 @@ def format_task_related_prompt(question, question_type, meta_data = None, perspe
             suffix = "Here are the options of actions you are selecting:\n" + suffix 
             ret = prefix + suffix
 
-    elif question_type == "direct_narration":
-
-        if learn_neighbor_actions == "prior" and meta_data and random.random() < 0.1:
-            ret = f"{perspective_prefix} {prev2_offset} seconds ago, you started an action {prev2_narration}. {prev1_offset} seconds ago, you started an action {prev1_narration}. What action are you currently performing? Give a short sentence such as 'move knife'. "            
-        else:            
-            ret = f"{perspective_prefix} What action are you performing? Give a short sentence such as 'move knife'."
+    elif question_type == "direct_narration":                  
+        ret = f"{perspective_prefix} What action are you performing? Give a short sentence such as 'move knife'."
 
     elif question_type == "temporal_detection":
         ret = question

llava/action/utils.py

@@ -1289,12 +1289,12 @@ def _get_item(self, i) -> Dict[str, torch.Tensor]:
                     start_timestamp = round(float(self.list_data_dict[i]['start_timestamp']), 2)
                     end_timestamp = round(float(self.list_data_dict[i]['end_timestamp']), 2)
                     uid = f"{vid}_{start_timestamp}_{end_timestamp}"
-                    # if True:
-                    #     meta_data = self.train_triple_lookup_narration.get(uid, None)
-                    if 'official_key' in sources[0]['question_type']:
-                        meta_data = self.train_triple_lookup_official.get(uid, None)
-                    elif 'GT_random_narration' in sources[0]['question_type']:
+                    if True:
                         meta_data = self.train_triple_lookup_narration.get(uid, None)
+                    # if 'official_key' in sources[0]['question_type']:
+                    #     meta_data = self.train_triple_lookup_official.get(uid, None)
+                    # elif 'GT_random_narration' in sources[0]['question_type']:
+                    #     meta_data = self.train_triple_lookup_narration.get(uid, None)
 
 
                 if 'EK100' not in video_file and 'EKframes' not in video_folder: