update video code

Author: ZhangYuanhan-AI

llava/model/builder.py

@@ -238,6 +238,11 @@ def load_from_hf(repo_id, filename, subfolder=None):
                             llava_cfg.delay_load = True  # a workaround for correctly loading v1.5 models
                     else:
                         llava_cfg = customized_config
+
+                    if overwrite_config is not None:
+                        rank0_print(f"Overwriting config with {overwrite_config}")
+                        for k, v in overwrite_config.items():
+                            setattr(llava_cfg, k, v)
                     model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, attn_implementation=attn_implementation, config=llava_cfg, **kwargs)
                 except:
                     raise ValueError(f"Model {model_name} not supported")

llava/model/llava_arch.py

@@ -223,7 +223,7 @@ def prepare_inputs_labels_for_multimodal(self, input_ids, position_ids, attentio
 
         if isinstance(modalities, str):
             modalities = [modalities]
-            
+
         if type(images) is list or images.ndim == 5:
             if type(images) is list:
                 images = [x.unsqueeze(0) if x.ndim == 3 else x for x in images]
@@ -242,6 +242,7 @@ def prepare_inputs_labels_for_multimodal(self, input_ids, position_ids, attentio
                 else:
                     images_list.append(image.unsqueeze(0))
 
+            # import pdb;pdb.set_trace()
             concat_images = torch.cat([image for image in images_list], dim=0)
             split_sizes = [image.shape[0] for image in images_list]
             encoded_image_features = self.encode_images(concat_images)