Merge pull request #195 from zucchini-nlp/main

Fix: videos in LLaVa-OV

Author: Luodian

docs/LLaVA_OneVision_Tutorials.ipynb

@@ -345,6 +345,7 @@
     "\n",
     "input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors=\"pt\").unsqueeze(0).to(device)\n",
     "image_sizes = [frame.size for frame in video_frames]\n",
+    "modalities = [\"video\"] * len(video_frames)\n",
     "\n",
     "# Generate response\n",
     "cont = model.generate(\n",
@@ -354,7 +355,7 @@
     "    do_sample=False,\n",
     "    temperature=0,\n",
     "    max_new_tokens=4096,\n",
-    "    modalities=[\"video\"],\n",
+    "    modalities=modalities,\n",
     ")\n",
     "text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)\n",
     "print(text_outputs[0])"