updates

Author: yeshaokai

action/dataset.py

@@ -630,7 +630,8 @@ def get_args_parser():
     ## avaion refinement 
     parser.add_argument('--action_predictions', default=None, type=str, help='path to action predictions')
     parser.add_argument('--topk_predictions', default = 5, type =int)
-
+    parser.add_argument('--llava_checkpoint', default = None, type = str)
+    
     return parser
 
 def prepare_llava(pretrained):
@@ -639,7 +640,6 @@ def prepare_llava(pretrained):
     from llava.model.builder import load_pretrained_model    
     warnings.filterwarnings("ignore")
     # Load the OneVision model
-    #pretrained = f"lmms-lab/llava-onevision-qwen2-{llm_size}-ov"
     model_name = "llava_qwen"
 
     device_map = "auto"
@@ -722,6 +722,9 @@ def get_topk_predictions(data, idx,  k):
 
     pretrained = f"lmms-lab/llava-onevision-qwen2-{args.llm_size}-ov"
 
+    if args.llava_checkpoint:
+        pretrained = args.llava_checkpoint
+    
     tokenizer, model, image_processor, max_length = prepare_llava(pretrained)
 
     if args.action_predictions:

llava/model/builder.py

@@ -200,6 +200,7 @@ def load_from_hf(repo_id, filename, subfolder=None):
                 model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, attn_implementation=attn_implementation, config=llava_cfg, **kwargs)
 
             elif "qwen" in model_name.lower() or "quyen" in model_name.lower():
+
                 tokenizer = AutoTokenizer.from_pretrained(model_path)
                 if "moe" in model_name.lower() or "A14B" in model_name.lower():
                     from llava.model.language_model.llava_qwen_moe import LlavaQwenMoeConfig
@@ -214,6 +215,7 @@ def load_from_hf(repo_id, filename, subfolder=None):
 
                 else:
                     from llava.model.language_model.llava_qwen import LlavaQwenConfig
+
                     if overwrite_config is not None:
                         llava_cfg = LlavaQwenConfig.from_pretrained(model_path)
                         rank0_print(f"Overwriting config with {overwrite_config}")

llava/model/language_model/llava_qwen.py

@@ -48,7 +48,10 @@ class LlavaQwenForCausalLM(Qwen2ForCausalLM, LlavaMetaForCausalLM):
 
     def __init__(self, config):
         # super(Qwen2ForCausalLM, self).__init__(config)
+        print ('what does config look like')
+        print (config)
         Qwen2ForCausalLM.__init__(self, config)
+        
         config.model_type = "llava_qwen"
         config.rope_scaling = None
 

run_EK100.sh

@@ -4,5 +4,6 @@ python3 action/dataset.py \
     --val-metadata /media/data/haozhe/VFM/EK100/epic-kitchens-100-annotations/EPIC_100_validation.csv \
     --llm_size 7b \
     --llava_num_frames 16 > kitchen_test.out 2>&1 \
+    # --llava_checkpoint /data/epic_kitchen/EK100_test/checkpoint-8402						    
     # --action_predictions action/avaion_predictions.json \
     # --topk_predictions 10