better benchmark code

Author: Ye Shaokai

llava/action/benchmark.py

@@ -3,20 +3,27 @@
 # benchmark gpt-4o on random_mcq_top5_500
 from llava.action.chatgpt_utils import GPTInferenceAnnotator
 
-root = '/data/EK100/EK100_320p_15sec_30fps_libx264'
-annotation_file = '/data/epic_kitchen/epic-kitchens-100-annotations/EPIC_100_validation.csv'
-avion_prediction_file = '/data/epic_kitchen/AVION_PREDS/avion_pred_ids_val.json'
-tim_prediction_file = '/data/epic_kitchen/TIM_PREDS/tim_pred_ids_val.json'
-n_frames = 4
+# root = '/data/EK100/EK100_320p_15sec_30fps_libx264'
+# annotation_file = '/data/epic_kitchen/epic-kitchens-100-annotations/EPIC_100_validation.csv'
+# avion_prediction_file = '/data/epic_kitchen/AVION_PREDS/avion_pred_ids_val.json'
+# tim_prediction_file = '/data/epic_kitchen/TIM_PREDS/tim_pred_ids_val.json'
+
+root = '/data/shaokai/EK100/'
+annotation_file = '/data/shaokai/epic-kitchens-100-annotations/EPIC_100_validation.csv'
+avion_prediction_file = '/data/shaokai/AVION_PREDS/avion_pred_ids_val.json'
+tim_prediction_file = '/data/shaokai/TIM_PREDS/tim_pred_ids_val.json'
+
+
+n_frames = 16
 topk = 5
 action_representation = 'GT_random_narration'
-#gpt_model = 'gpt-4o-mini-2024-07-18'
-gpt_model = 'gpt-4o-2024-08-06'
+gpt_model = 'gpt-4o-mini-2024-07-18'
+#gpt_model = 'gpt-4o-2024-08-06'
 perspective = 'first_person'
 benchmark_testing = True
 
 
-def benchmark_avion_mcq(n_samples):
+def benchmark_avion_mcq(n_samples, gpt_model):
 
     inferencer = GPTInferenceAnnotator(gpt_model,
                                        root,
@@ -29,9 +36,10 @@ def benchmark_avion_mcq(n_samples):
                                         perspective = perspective,
                                         benchmark_testing = benchmark_testing,
                                         topk = topk)
-    inferencer.multi_process_run(n_samples)
+    inferencer.multi_process_run(n_samples = n_samples,
+                                 offset = 0)
 
-def benchmark_tim_mcq(n_samples):
+def benchmark_tim_mcq(n_samples, gpt_model):
 
     inferencer = GPTInferenceAnnotator(gpt_model,
                                         root,
@@ -44,9 +52,9 @@ def benchmark_tim_mcq(n_samples):
                                         perspective = perspective,
                                         benchmark_testing = benchmark_testing,
                                         topk = topk) 
-    inferencer.multi_process_run(n_samples)    
+    inferencer.multi_process_run(n_samples = n_samples, offset = 0)    
 
-def benchmark_random_mcq(n_samples):
+def benchmark_random_mcq(n_samples, gpt_model):
     inferencer = GPTInferenceAnnotator(gpt_model,
                                        root,
                                        annotation_file,
@@ -59,10 +67,10 @@ def benchmark_random_mcq(n_samples):
                                         benchmark_testing = benchmark_testing,
                                         topk = topk) 
 
-    inferencer.multi_process_run(n_samples)
+    inferencer.multi_process_run(n_samples = n_samples, offset = 0)
 
 
 if __name__ == '__main__':
-    #benchmark_avion_mcq(100)
-    benchmark_tim_mcq(100)
-    #benchmark_random_mcq(100)    
+    benchmark_avion_mcq(-1)
+    benchmark_tim_mcq(-1)
+    benchmark_random_mcq(-1)

llava/action/chatgpt_utils.py

@@ -481,9 +481,8 @@ def multi_process_run(self, offset= 0, n_samples = -1, disable_api_calling = Fal
         if combined_results and 'mc_' in self.question_type:
             calculation = calculate_gpt_accuracy(data = combined_results)
 
-        prefix = self.gen_type
         assert n_samples != -1
-        checkpoint_name = f"{prefix}_{self.action_representation}_top{self.topk}_{self.clip_length}f_{n_samples}samples.json"
+        checkpoint_name = f"{self.gpt_model}_{self.gen_type}_{self.action_representation}_top{self.topk}_{self.clip_length}f_{n_samples}samples.json"
 
         if self.do_visualization:
             self.checkpoint(combined_results, os.path.join(self.vis_folder, checkpoint_name))