added assertion to check whether avion gt matches my gt

Author: Ye Shaokai

action/ek_eval.py

@@ -337,9 +337,18 @@ def prepare_llava(pretrained):
     model_name = "llava_qwen"
 
     device_map = "auto"
-    print ('pretrained???', pretrained)
-    #tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map, attn_implementation="sdpa")
-    tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, torch_dtype="bfloat16", device_map=device_map)  # Add any other thing you want to pass in llava_model_args
+
+    overwrite_config = None
+    if 'video' in pretrained:
+        overwrite_config =  {'tie_word_embeddings': False, 'use_cache': True, "vocab_size": 152064}
+
+    print ('overwrite config', overwrite_config)
+    tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, 
+                                                                          None, 
+                                                                          model_name, 
+                                                                          torch_dtype="bfloat16", 
+                                                                          device_map=device_map, 
+                                                                          overwrite_config = overwrite_config)  # Add any other thing you want to pass in llava_model_args
 
 
     return tokenizer, model, image_processor, max_length

action/generate_description.py

@@ -47,10 +47,12 @@ def generate_train_ann(ann_file, verb_ids, noun_ids, gen_type = 'naive', avion_p
         elif gen_type == "avion_mc":
             vn_str = f'{row[10]}:{row[12]}'
             avion_preds = avion_train_predictions[str(idx)]['predictions']
+            gt_from_avion = avion_train_predictions[str(idx)]['target']
             mc_data = mc_generator.generate_multi_choice(vn_str, avion_preds, n_options)
             options = mc_data['options'][0]
             gt_answer_letter = mc_data['gt_answer_letter'][0]
             gt_answer_name = mc_data['gt_answer_name'][0]
+            assert gt_answer_name.replace(' ', ':') == gt_from_avion
             conversation = generate_random_mc_conversation(options, gt_answer_letter, gt_answer_name )
 
         data = {'video': vid_path,

shaokai_generate_train.sh

@@ -8,5 +8,5 @@ python3 action/generate_description.py \
    --out_folder /storage-rcp-pure/upmwmathis_scratch/shaokai/EK100_inst_train \
    --avion_train_predictions /storage-rcp-pure/upmwmathis_scratch/shaokai/avion_predictions_train.json \
    --gen_type avion_mc \
-   --n_options 10
+   --n_options 3