Merge branch 'shaokai/dev' of github.com:yeshaokai/LLaVA-NeXT into shaokai/dev

Author: Ye Shaokai

llava/action/benchmark.py

@@ -10,9 +10,10 @@
 n_frames = 4
 topk = 5
 action_representation = 'GT_random_narration'
-gpt_model = 'gpt-4o-mini-2024-07-18'
-# gpt_model = 'gpt-4o-2024-08-06'
-perspective = 'third_person'
+#gpt_model = 'gpt-4o-mini-2024-07-18'
+gpt_model = 'gpt-4o-2024-08-06'
+perspective = 'first_person'
+benchmark_testing = True
 
 
 def benchmark_avion_mcq(n_samples):
@@ -26,6 +27,7 @@ def benchmark_avion_mcq(n_samples):
                                         question_type = 'mc_',
                                         action_representation=action_representation,
                                         perspective = perspective,
+                                        benchmark_testing = benchmark_testing,
                                         topk = topk)
     inferencer.multi_process_run(n_samples)
 
@@ -40,6 +42,7 @@ def benchmark_tim_mcq(n_samples):
                                         question_type = 'mc_',
                                         action_representation=action_representation,
                                         perspective = perspective,
+                                        benchmark_testing = benchmark_testing,
                                         topk = topk) 
     inferencer.multi_process_run(n_samples)    
 
@@ -53,6 +56,7 @@ def benchmark_random_mcq(n_samples):
                                         question_type = 'mc_',
                                         action_representation=action_representation,
                                         perspective = perspective,
+                                        benchmark_testing = benchmark_testing,
                                         topk = topk) 
 
     inferencer.multi_process_run(n_samples)
@@ -61,4 +65,4 @@ def benchmark_random_mcq(n_samples):
 if __name__ == '__main__':
     benchmark_avion_mcq(100)
     benchmark_tim_mcq(100)
-    benchmark_random_mcq(100)    
+    #benchmark_random_mcq(100)    

llava/action/chatgpt_utils.py

@@ -347,7 +347,8 @@ def __init__(self,
                  question_type = 'cot_mc',
                  debug = False,
                  topk = 10,
-                 perspective = 'first_person'
+                 perspective = 'first_person',
+                 benchmark_testing = False
                  ):
         """
         Parameters
@@ -370,6 +371,7 @@ def __init__(self,
 
         self.gen_type = gen_type
         self.perspective = perspective
+        self.benchmark_testing = benchmark_testing
         assert gen_type in ['avion', 'tim', 'random']
 
         if gen_type == 'avion' or gen_type == 'tim':                  
@@ -409,6 +411,7 @@ def init_data(self):
                                                             self.mapping_vn2narration,
                                                             self.verb_maps,
                                                             self.noun_maps,
+                                                            benchmark_testing = self.benchmark_testing,
                                                             is_train = False)
             else:
                 mc_data = self.mc_generator.generate_multi_choice(gt_vn,
@@ -420,6 +423,7 @@ def init_data(self):
                                                             self.mapping_vn2narration,
                                                             self.verb_maps,
                                                             self.noun_maps,
+                                                            benchmark_testing = self.benchmark_testing,
                                                             is_train = False)
 
             options = mc_data['options'][0]

llava/action/dataset.py

@@ -313,7 +313,8 @@ def __getitem__(self, i):
                                                         self.mapping_vn2narration,                                                        
                                                         self.verb_maps, 
                                                         self.noun_maps,
-                                                        is_train = False) # note we only use this dataset for evaluation for now.
+                                                        is_train = False,
+                                                        benchmark_testing = eval_args.benchmark_testing) # note we only use this dataset for evaluation for now.
 
 
         return frames, data, time_meta, i

llava/action/ek_eval.py

@@ -129,6 +129,7 @@ def get_args_parser():
     parser.add_argument('--pseudo_folder', default = None, type = str)
     parser.add_argument('--output_dir', default = None, type = str)
     parser.add_argument("--perspective", default = "first_person", type = str)
+    parser.add_argument('--benchmark_testing', action='store_true', default = False)
     return parser
 
 def prepare_llava(pretrained):

llava/action/utils.py

@@ -411,7 +411,8 @@ def generate_multi_choice(self,
                               mapping_vn2narration, 
                               verb_maps, 
                               noun_maps,
-                              is_train = True
+                              is_train = True,
+                              benchmark_testing = False
                               ):
 
         """
@@ -425,7 +426,7 @@ def generate_multi_choice(self,
         if is_train:
             return self.train_generate(gt_vn, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps)
         else:
-            return self.test_generate(gt_vn, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps)
+            return self.test_generate(gt_vn, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps, benchmark_testing = benchmark_testing)
 
     def train_generate(self, gt_vn, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps):
         # letters as A, B, C, D, .. Note we maximally support 26 letters
@@ -522,12 +523,34 @@ def train_generate(self, gt_vn, avion_predictions, narration, k, action_represen
             }  
         return mc_data              
 
-    def test_generate(self, gt_vn, avion_predictions, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps):
+    def test_generate(self, 
+                      gt_vn, 
+                      action_model_predictions, 
+                      narration, 
+                      k, 
+                      action_representation, 
+                      n_narrations, 
+                      labels, 
+                      mapping_vn2narration, 
+                      verb_maps, 
+                      noun_maps,
+                      benchmark_testing = False
+                      ):
         """
         During testing, we use the top k predictions from avion. No randomness. We do not mix the gt_vn with the avion predictions
         """        
-
-        answer_ids = avion_predictions[:k]
+        answer_ids = action_model_predictions[:k]
+        
+        if benchmark_testing:
+            # if we are testing on benchmark, we need to ensure that the gt_vn is in the top k predictions
+            # if not, we remove the last prediction and add the gt_vn
+            if gt_vn not in answer_ids:
+                answer_ids.pop()
+                answer_ids.append(gt_vn)
+       
+        # let's shuffle answer_ids so that the gt_vn is not always at the end
+        random.shuffle(answer_ids)         
+        
         answers = []
         for answer_id in answer_ids:
             answer = parse_vn_ids(answer_id, gt_vn, narration, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps)
@@ -566,7 +589,8 @@ def generate_multi_choice(self,
                               mapping_vn2narration, 
                               verb_maps, 
                               noun_maps,
-                              is_train = True
+                              is_train = True,
+                              benchmark_testing = False
                               ):
         """
         Generate k multiple choices from gt_vn pairs
@@ -578,7 +602,7 @@ def generate_multi_choice(self,
         if is_train:
             return self.train_generate(gt_vn, avion_predictions, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps)
         else:
-            return self.test_generate(gt_vn, avion_predictions, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps)
+            return self.test_generate(gt_vn, avion_predictions, narration, k, action_representation, n_narrations, labels, mapping_vn2narration, verb_maps, noun_maps, benchmark_testing = benchmark_testing)
 
 def get_frame_ids(start_frame, end_frame, num_segments=32, jitter=True):
     frame_ids = np.convolve(np.linspace(start_frame, end_frame, num_segments + 1), [0.5, 0.5], mode='valid')

llava/train/train.py

@@ -204,6 +204,7 @@ class EK100EvalArguments:
     learn_neighbor_actions: bool = False
     perspective: str = "first_person"
     pseudo_folder: str = ""
+    benchmark_testing: bool = False
 
 def maybe_zero_3(param, ignore_status=False, name=None):
     from deepspeed import zero