updates

Author: Ye Shaokai

action/ek_eval.py

@@ -390,7 +390,10 @@ def get_topk_predictions(data, idx,  k):
 
     return mc_data, predictions[0]
 
-def evaluate_on_EK100(eval_args, model= None, tokenizer= None, max_length= None, image_processor= None):
+def evaluate_on_EK100(eval_args, 
+                      model= None, 
+                      tokenizer= None, 
+                      image_processor= None):
 
     if image_processor is None:
         image_processor = model.get_vision_tower().image_processor
@@ -474,7 +477,7 @@ def evaluate_on_EK100(eval_args, model= None, tokenizer= None, max_length= None,
         if finish_early and idx>999:
             break
 
-        pred = llava_inference(frames, tokenizer, model, image_processor, max_length, mc_data,  clip_length = eval_args.clip_length, num_frames=eval_args.llava_num_frames)
+        pred = llava_inference(frames, tokenizer, model, image_processor,  mc_data,  clip_length = eval_args.clip_length, num_frames=eval_args.llava_num_frames)
 
         # if valid letter is found in the prediction, then we will use that as the prediction
         rank0_print ('llava pred', pred, 'avion_pred', avion_pred, 'gt_name', gt_name)        

action/generate_description.py

@@ -12,7 +12,7 @@ def datetime2sec(str):
     hh, mm, ss = str.split(':')
     return int(hh) * 3600 + int(mm) * 60 + float(ss)
 
-def generate_train_ann(ann_file, verb_ids, noun_ids, gen_type = 'naive', avion_prediction_path = ''):
+def generate_train_ann(ann_file, verb_ids, noun_ids, gen_type = 'naive', avion_prediction_path = '', n_options = 5):
     assert gen_type in GEN_TYPES
     # epic kitchen uses csv
     csv_reader = csv.reader(open(ann_file))
@@ -39,15 +39,15 @@ def generate_train_ann(ann_file, verb_ids, noun_ids, gen_type = 'naive', avion_p
         elif gen_type == "random_mc":
             # here we use the index
             vn_str = f'{row[10]}:{row[12]}'
-            mc_data = mc_generator.generate_multi_choice(vn_str, 5)
+            mc_data = mc_generator.generate_multi_choice(vn_str, n_options)
             options = mc_data['option'][0]
             gt_answer_letter = mc_data['gt_answer_letter'][0]
             gt_answer_name = mc_data['gt_answer_name'][0]
             conversation = generate_random_mc_conversation(options, gt_answer_letter, gt_answer_name )
         elif gen_type == "avion_mc":
             vn_str = f'{row[10]}:{row[12]}'
             avion_preds = avion_train_predictions[str(idx)]['predictions']
-            mc_data = mc_generator.generate_multi_choice(vn_str, avion_preds, 5)
+            mc_data = mc_generator.generate_multi_choice(vn_str, avion_preds, n_options)
             options = mc_data['option'][0]
             gt_answer_letter = mc_data['gt_answer_letter'][0]
             gt_answer_name = mc_data['gt_answer_name'][0]
@@ -86,27 +86,30 @@ def get_args():
     parser.add_argument('--out_folder', default = '/data/shaokai/EK100_in_LLAVA/', type = str)
     parser.add_argument('--avion_train_predictions', default = '/data/shaokai/avion_predictions_train.json', type = str)
     parser.add_argument('--gen_type', default = 'avion_mc', type = str, choices = GEN_TYPES)
+    parser.add_argument('--n_options', default = 5, type = int)
     return parser.parse_args()
 
 def main(): 
     args = get_args()    
     ann_file = args.train_metadata
-    inst_train_folder = os.path.join(args.out_folder, args.gen_type)
+    inst_train_folder = os.path.join(args.out_folder, f'{args.gen_type}_top{args.n_options}')
 
     print ('train_metadata', args.train_metadata)
     print ('out_folder', args.out_folder)
     print ('loading predictions from ', args.avion_train_predictions)
     print ('gen_type is ', args.gen_type)
+    print ('n_options', args.n_options)
 
-    os.makedirs(inst_train_folder, exist_ok=True)
+    os.makedirs(inst_train_folder, exist_ok=True)    
 
     anno_path = Path(ann_file).parent
     _, _, verb_ids, noun_ids = generate_label_map(anno_path)
     conv_lst = generate_train_ann(ann_file, 
                                   verb_ids, 
                                   noun_ids, 
                                   gen_type = args.gen_type, 
-                                  avion_prediction_path = args.avion_train_predictions)
+                                  avion_prediction_path = args.avion_train_predictions,
+                                  n_options = args.n_options)
 
     # save it to a jsonl
     with open(os.path.join(inst_train_folder,'train_convs_narration.jsonl'), 'w') as f:

action/llava_ov_inference.py

@@ -18,7 +18,6 @@ def llava_inference(video_frames,
     tokenizer, 
     model, 
     image_processor, 
-    max_length, 
     mc_data,
     clip_length = 16,
     num_frames=16):
@@ -29,7 +28,6 @@ def llava_inference(video_frames,
     temporal_stride = clip_length // num_frames
     video_frames = video_frames[::temporal_stride]
     image_tensors = []
-    #frames = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"].half().cuda()
     frames = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"].cuda().to(torch.bfloat16)
     image_tensors.append(frames)
 

llava/train/llava_trainer.py

@@ -251,7 +251,7 @@ def __init__(self, *args, tokenizer = None, eval_args = None, model_max_length =
     def evaluate(self, eval_dataset=None, ignore_keys=None, metric_key_prefix="eval"):
         from action.ek_eval import evaluate_on_EK100        
 
-        accuracy = evaluate_on_EK100(self.eval_args, self.model, self.tokenizer,  self.model_max_length)
+        accuracy = evaluate_on_EK100(self.eval_args, self.model, self.tokenizer)
 
         metrics = {f"{metric_key_prefix}_EK100_accuracy": accuracy}
 

shaokai_generate_train.sh

@@ -8,4 +8,5 @@ python3 action/generate_description.py \
    --out_folder /storage-rcp-pure/upmwmathis_scratch/shaokai/EK100_inst_train \
    --avion_train_predictions /storage-rcp-pure/upmwmathis_scratch/shaokai/avion_predictions_train.json \
    --gen_type avion_mc \
+   --n_options 10