update video code

Author: ZhangYuanhan-AI

llava/model/llava_arch.py

@@ -221,6 +221,9 @@ def prepare_inputs_labels_for_multimodal(self, input_ids, position_ids, attentio
         if vision_tower is None or images is None or input_ids.shape[1] == 1:
             return input_ids, position_ids, attention_mask, past_key_values, None, labels
 
+        if isinstance(modalities, str):
+            modalities = [modalities]
+            
         if type(images) is list or images.ndim == 5:
             if type(images) is list:
                 images = [x.unsqueeze(0) if x.ndim == 3 else x for x in images]
@@ -230,6 +233,8 @@ def prepare_inputs_labels_for_multimodal(self, input_ids, position_ids, attentio
                 if modalities[_] == "video":
                     video_idx_in_batch.append(_)
 
+            # print(video_idx_in_batch)
+
             images_list = []
             for image in images:
                 if image.ndim == 4:

playground/demo/video_demo.py

@@ -1,11 +1,11 @@
 import argparse
 import torch
 
-from llavavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llavavid.conversation import conv_templates, SeparatorStyle
-from llavavid.model.builder import load_pretrained_model
-from llavavid.utils import disable_torch_init
-from llavavid.mm_utils import process_anyres_image,tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from llava.conversation import conv_templates, SeparatorStyle
+from llava.model.builder import load_pretrained_model
+from llava.utils import disable_torch_init
+from llava.mm_utils import process_anyres_image,tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 import json
 import os

playground/demo/xU25MMA2N4aVtYay.mp4

@@ -87,7 +87,7 @@ echo $SAVE_DIR
 
 # wait
 
-python3 llavavid/eval/eval_activitynet_qa.py \
+python3 llava/eval/eval_activitynet_qa.py \
     --pred_path ./work_dirs/eval_activitynet/$SAVE_DIR \
     --output_dir ./work_dirs/eval_activitynet/$SAVE_DIR/results \
     --output_json ./work_dirs/eval_activitynet/$SAVE_DIR/results.json \