runable version on AMGm0

Author: HaozheQi

.gitignore

@@ -15,7 +15,6 @@ dist
 # Editor
 .idea
 *.swp
-.vscode
 
 # Other
 .DS_Store
@@ -71,3 +70,7 @@ playground/*.json
 mlx_configs/
 data_processing/
 # demo/
+
+
+experiments/
+*.out

.vscode/launch.json

@@ -0,0 +1,101 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Run LLAVA Training with torchrun",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "torch.distributed.run",
+            "env": {
+                "CUDA_VISIBLE_DEVICES": "1,2",
+                "OMP_NUM_THREADS": "8",
+                "NCCL_IB_DISABLE": "0",
+                "NCCL_IB_GID_INDEX": "3",
+                "NCCL_SOCKET_IFNAME": "eth0",
+                "NCCL_DEBUG": "INFO",
+                "ACCELERATE_CPU_AFFINITY": "1",
+                "LD_PRELOAD": "/usr/lib/x86_64-linux-gnu/libffi.so.7",
+            },
+            "args": [
+                "--nproc_per_node=2",
+                "--nnodes=1",
+                "--node_rank=0",
+                "--master_addr=127.0.0.1",
+                "--master_port=29500",
+                "llava/train/train_mem.py",
+                "--deepspeed", "scripts/zero3.json",
+                "--model_name_or_path", "lmms-lab/llava-onevision-qwen2-0.5b-ov",
+                "--version", "qwen_1_5",
+                "--data_path", "scripts/train/onevision.yaml",
+                // "--image_folder", "/mediaPFM/data/haozhe/onevision/llava_data",
+                "--image_folder", "/mediaPFM/data/haozhe/onevision/llava_data/geo3k/",
+                "--video_folder", "/mediaPFM/data/haozhe/onevision/llava_video",
+                "--mm_tunable_parts", "mm_vision_tower,mm_mlp_adapter,mm_language_model",
+                "--mm_vision_tower_lr", "2e-6",
+                "--vision_tower", "google/siglip-so400m-patch14-384",
+                "--mm_projector_type", "mlp2x_gelu",
+                "--mm_vision_select_layer", "-2",
+                "--mm_use_im_start_end", "False",
+                "--mm_use_im_patch_token", "False",
+                "--group_by_modality_length", "True",
+                "--image_aspect_ratio", "anyres_max_9",
+                "--image_grid_pinpoints", "(1x1),...,(6x6)",
+                "--mm_patch_merge_type", "spatial_unpad",
+                "--bf16", "True",
+                "--run_name", "test",
+                "--output_dir", "experiments/test",
+                "--num_train_epochs", "1",
+                "--per_device_train_batch_size", "1",
+                "--per_device_eval_batch_size", "4",
+                "--gradient_accumulation_steps", "2",
+                "--evaluation_strategy", "no",
+                "--save_strategy", "steps",
+                "--save_steps", "1000",
+                "--save_total_limit", "1",
+                "--learning_rate", "1e-5",
+                "--weight_decay", "0.",
+                "--warmup_ratio", "0.03",
+                "--lr_scheduler_type", "cosine",
+                "--logging_steps", "1",
+                "--tf32", "True",
+                "--model_max_length", "32768",
+                "--gradient_checkpointing", "True",
+                "--dataloader_num_workers", "4",
+                "--lazy_preprocess", "True",
+                "--report_to", "wandb",
+                "--torch_compile", "True",
+                "--torch_compile_backend", "inductor",
+                "--dataloader_drop_last", "True",
+                "--frames_upbound", "32",
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}
+
+
+// {
+//     // Use IntelliSense to learn about possible attributes.
+//     // Hover to view descriptions of existing attributes.
+//     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+//     "version": "0.2.0",
+//     "configurations": [
+//         {
+//             "name": "Python: Current File",
+//             "type": "debugpy",
+//             "request": "launch",
+//             "program": "docs/LLaVA_OneVision_Tutorials.py",
+//             "console": "integratedTerminal",
+//             "env":{"CUDA_VISIBLE_DEVICES":"0",
+//                    "LD_PRELOAD": "/usr/lib/x86_64-linux-gnu/libffi.so.7",
+//                    "LD_LIBRARY_PATH": "/home/haozhe/miniconda3/envs/llava/lib"},
+//             "justMyCode": false,
+//             // "args": [
+//             //     "--run_dir_name", "test",
+//             //     // "--use_big_decoder"
+//             // ]
+//         }
+//     ]
+// }

add_dataset_name.py

@@ -0,0 +1,33 @@
+import json
+import os
+
+json_root = '/mediaPFM/data/haozhe/onevision/llava_instruct_old'
+save_root = '/mediaPFM/data/haozhe/onevision/llava_instruct'
+
+json_list = os.listdir(json_root)
+for json_name in json_list:
+    json_path = os.path.join(json_root, json_name)
+    if json_path.endswith(".jsonl"):
+        cur_data_dict = []
+        with open(json_path, "r") as json_file:
+            for line in json_file:
+                cur_data_dict.append(json.loads(line.strip()))
+    elif json_path.endswith(".json"):
+        with open(json_path, "r") as json_file:
+            cur_data_dict = json.load(json_file)
+    else:
+        raise ValueError(f"Unsupported file type: {json_path}")
+    
+    dataset_name = json_path.split('/')[-1].split('.')[0]
+    for data in cur_data_dict:
+        data['dataset_name'] = dataset_name
+
+    # save back
+    save_path = os.path.join(save_root, json_name)
+    with open(save_path, "w") as json_file:
+        if json_path.endswith(".jsonl"):
+            for data in cur_data_dict:
+                json_file.write(json.dumps(data) + "\n")
+        elif json_path.endswith(".json"):
+            json.dump(cur_data_dict, json_file, indent=4)
+    aa = 1

docs/LLaVA_OneVision_Tutorials.ipynb

@@ -24,9 +24,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ImportError",
+     "evalue": "cannot import name 'LlavaLlamaForCausalLM' from 'llava.model' (/media1/data/haozhe/VFM/LLaVA-NeXT/llava/model/__init__.py)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllava\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbuilder\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_pretrained_model\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllava\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmm_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_model_name_from_path, process_images, tokenizer_image_token\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllava\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconstants\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX\n",
+      "File \u001b[0;32m/media1/data/haozhe/VFM/LLaVA-NeXT/llava/model/builder.py:24\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllava\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllava\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconstants\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\n\u001b[0;32m---> 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllava\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m rank0_print\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_pretrained_model\u001b[39m(model_path, model_base, model_name, load_8bit\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, load_4bit\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, device_map\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m\"\u001b[39m, attn_implementation\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mflash_attention_2\u001b[39m\u001b[38;5;124m\"\u001b[39m, customized_config\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, overwrite_config\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     28\u001b[0m     kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdevice_map\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m device_map\n",
+      "File \u001b[0;32m/media1/data/haozhe/VFM/LLaVA-NeXT/llava/__init__.py:1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m LlavaLlamaForCausalLM\n",
+      "\u001b[0;31mImportError\u001b[0m: cannot import name 'LlavaLlamaForCausalLM' from 'llava.model' (/media1/data/haozhe/VFM/LLaVA-NeXT/llava/model/__init__.py)"
+     ]
+    }
+   ],
    "source": [
     "from llava.model.builder import load_pretrained_model\n",
     "from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token\n",

docs/LLaVA_OneVision_Tutorials.py

@@ -0,0 +1,125 @@
+# from llava.model.builder import load_pretrained_model
+# from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token
+# from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
+# from llava.conversation import conv_templates, SeparatorStyle
+
+# from PIL import Image
+# import requests
+# import copy
+# import torch
+
+# import sys
+# import warnings
+
+
+
+# warnings.filterwarnings("ignore")
+# pretrained = "lmms-lab/llava-onevision-qwen2-0.5b-si"
+# model_name = "llava_qwen"
+# device = "cuda"
+# device_map = "auto"
+# tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map)  # Add any other thing you want to pass in llava_model_args
+
+# model.eval()
+
+# url = "https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true"
+# image = Image.open(requests.get(url, stream=True).raw)
+# image_tensor = process_images([image], image_processor, model.config)
+# image_tensor = [_image.to(dtype=torch.float16, device=device) for _image in image_tensor]
+
+# conv_template = "qwen_1_5"  # Make sure you use correct chat template for different models
+# question = DEFAULT_IMAGE_TOKEN + "\nWhat is shown in this image?"
+# conv = copy.deepcopy(conv_templates[conv_template])
+# conv.append_message(conv.roles[0], question)
+# conv.append_message(conv.roles[1], None)
+# prompt_question = conv.get_prompt()
+
+# input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+# image_sizes = [image.size]
+
+
+# cont = model.generate(
+#     input_ids,
+#     images=image_tensor,
+#     image_sizes=image_sizes,
+#     do_sample=False,
+#     temperature=0,
+#     max_new_tokens=4096,
+# )
+# text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+# print(text_outputs)
+
+
+
+
+from operator import attrgetter
+from llava.model.builder import load_pretrained_model
+from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token
+from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
+from llava.conversation import conv_templates, SeparatorStyle
+
+import torch
+import cv2
+import numpy as np
+from PIL import Image
+import requests
+import copy
+import warnings
+from decord import VideoReader, cpu
+
+warnings.filterwarnings("ignore")
+# Load the OneVision model
+pretrained = "lmms-lab/llava-onevision-qwen2-7b-ov"
+model_name = "llava_qwen"
+device = "cuda"
+device_map = "auto"
+tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map, attn_implementation="sdpa")
+
+model.eval()
+
+
+# Function to extract frames from video
+def load_video(video_path, max_frames_num):
+    if type(video_path) == str:
+        vr = VideoReader(video_path, ctx=cpu(0))
+    else:
+        vr = VideoReader(video_path[0], ctx=cpu(0))
+    total_frame_num = len(vr)
+    uniform_sampled_frames = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int)
+    frame_idx = uniform_sampled_frames.tolist()
+    spare_frames = vr.get_batch(frame_idx).asnumpy()
+    return spare_frames  # (frames, height, width, channels)
+
+
+# Load and process video
+video_path = "docs/jobs.mp4"
+video_frames = load_video(video_path, 16)
+print(video_frames.shape) # (16, 1024, 576, 3)
+image_tensors = []
+frames = image_processor.preprocess(video_frames, return_tensors="pt")["pixel_values"].half().cuda()
+image_tensors.append(frames)
+
+# Prepare conversation input
+conv_template = "qwen_1_5"
+question = f"{DEFAULT_IMAGE_TOKEN}\nDescribe what's happening in this video."
+
+conv = copy.deepcopy(conv_templates[conv_template])
+conv.append_message(conv.roles[0], question)
+conv.append_message(conv.roles[1], None)
+prompt_question = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
+image_sizes = [frame.size for frame in video_frames]
+
+# Generate response
+cont = model.generate(
+    input_ids,
+    images=image_tensors,
+    image_sizes=image_sizes,
+    do_sample=False,
+    temperature=0,
+    max_new_tokens=4096,
+    modalities=["video"],
+)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+print(text_outputs[0])

docs/download_data.py

@@ -0,0 +1,64 @@
+import os
+from datasets import load_dataset
+from tqdm import tqdm
+import json
+import yaml
+
+avaliable_datasets = ['CLEVR-Math(MathV360K)', 'FigureQA(MathV360K)', 'GEOS(MathV360K)', 'GeoQA+(MathV360K)', 
+                      'Geometry3K(MathV360K)', 'IconQA(MathV360K)', 'MapQA(MathV360K)', 'PMC-VQA(MathV360K)', 
+                      'Super-CLEVR(MathV360K)', 'TabMWP(MathV360K)', 'UniGeo(MathV360K)', 'VisualWebInstruct(filtered)', 
+                      'VizWiz(MathV360K)', 'ai2d(cauldron,llava_format)', 'ai2d(gpt4v)', 'ai2d(internvl)', 
+                      'allava_instruct_laion4v', 'allava_instruct_vflan4v', 'aokvqa(cauldron,llava_format)', 
+                      'chart2text(cauldron)', 'chartqa(cauldron,llava_format)', 'chrome_writting', 
+                      'clevr(cauldron,llava_format)', 'diagram_image_to_text(cauldron)', 'dvqa(cauldron,llava_format)', 
+                      'figureqa(cauldron,llava_format)', 'geo170k(align)', 'geo170k(qa)', 'geo3k', 'geomverse(cauldron)', 
+                      'hateful_memes(cauldron,llava_format)', 'hitab(cauldron,llava_format)', 'hme100k', 
+                      'iam(cauldron)', 'iconqa(cauldron,llava_format)', 'iiit5k', 'image_textualization(filtered)', 
+                      'infographic(gpt4v)', 'infographic_vqa', 'infographic_vqa_llava_format', 
+                      'intergps(cauldron,llava_format)', 'k12_printing', 'llavar_gpt4_20k', 'lrv_chart', 
+                      'lrv_normal(filtered)', 'magpie_pro(l3_80b_mt)', 'magpie_pro(l3_80b_st)', 
+                      'magpie_pro(qwen2_72b_st)', 'mapqa(cauldron,llava_format)', 'mathqa', 'mavis_math_metagen', 
+                      'mavis_math_rule_geo', 'multihiertt(cauldron)', 'orand_car_a', 'raven(cauldron)', 
+                      'rendered_text(cauldron)', 'robut_sqa(cauldron)', 'robut_wikisql(cauldron)', 
+                      'robut_wtq(cauldron,llava_format)', 'scienceqa(cauldron,llava_format)', 'scienceqa(nona_context)', 
+                      'screen2words(cauldron)', 'sharegpt4o', 'sharegpt4v(coco)', 'sharegpt4v(knowledge)', 
+                      'sharegpt4v(llava)', 'sharegpt4v(sam)', 'sroie', 'st_vqa(cauldron,llava_format)', 
+                      'tabmwp(cauldron)', 'tallyqa(cauldron,llava_format)', 'textcaps', 'textocr(gpt4v)', 
+                      'tqa(cauldron,llava_format)', 'ureader_cap', 'ureader_ie', 'vision_flan(filtered)', 
+                      'vistext(cauldron)', 'visual7w(cauldron,llava_format)', 'visualmrc(cauldron)', 
+                      'vqarad(cauldron,llava_format)', 'vsr(cauldron,llava_format)', 'websight(cauldron)']
+
+chossen_datasets = ['sharegpt4v(sam)', 'sharegpt4v(llava)']
+
+image_base = "/mediaPFM/data/haozhe/onevision/llava_data"
+json_base = "/mediaPFM/data/haozhe/onevision/llava_instruct"
+dataset_yaml = 'scripts/train/onevision.yaml'
+
+# # open the yaml file
+# with open(dataset_yaml, 'r') as f:
+#     dataset_config = yaml.safe_load(f)
+
+# dataset_paths = {}
+# for data_info in dataset_config['datasets']:
+#     dataset_paths[data_info['json_path'].split('/')[-1]] = data_info['json_path']
+
+
+for dataset_name in chossen_datasets:
+    data = load_dataset("lmms-lab/LLaVA-OneVision-Data", dataset_name, split="train")
+    converted_data = []
+
+    image_folder = os.path.join(image_base, dataset_name)
+    os.makedirs(image_folder, exist_ok=True)
+
+    for da in tqdm(data):
+        json_data = {}
+        json_data["id"] = da["id"]
+        if da["image"] is not None:
+            json_data["image"] = f"{da['id']}.png"
+            da["image"].save(os.path.join(image_folder, json_data["image"]))
+        json_data["conversations"] = da["conversations"]
+        converted_data.append(json_data)
+
+
+    with open(os.path.join(json_base, '{}.json'.format(dataset_name)), "w") as f:
+        json.dump(converted_data, f, indent=4, ensure_ascii=False)