Merge branch 'main' of github.com:HaozheQi/LLaVA-NeXT into main

Author: yeshaokai

.gitignore

@@ -15,7 +15,6 @@ dist
 # Editor
 .idea
 *.swp
-.vscode
 
 # Other
 .DS_Store
@@ -71,3 +70,7 @@ playground/*.json
 mlx_configs/
 data_processing/
 # demo/
+
+
+experiments/
+*.out

.vscode/launch.json

@@ -0,0 +1,101 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Run LLAVA Training with torchrun",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "torch.distributed.run",
+            "env": {
+                "CUDA_VISIBLE_DEVICES": "1,2",
+                "OMP_NUM_THREADS": "8",
+                "NCCL_IB_DISABLE": "0",
+                "NCCL_IB_GID_INDEX": "3",
+                "NCCL_SOCKET_IFNAME": "eth0",
+                "NCCL_DEBUG": "INFO",
+                "ACCELERATE_CPU_AFFINITY": "1",
+                "LD_PRELOAD": "/usr/lib/x86_64-linux-gnu/libffi.so.7",
+            },
+            "args": [
+                "--nproc_per_node=2",
+                "--nnodes=1",
+                "--node_rank=0",
+                "--master_addr=127.0.0.1",
+                "--master_port=29500",
+                "llava/train/train_mem.py",
+                "--deepspeed", "scripts/zero3.json",
+                "--model_name_or_path", "lmms-lab/llava-onevision-qwen2-0.5b-ov",
+                "--version", "qwen_1_5",
+                "--data_path", "scripts/train/onevision.yaml",
+                // "--image_folder", "/mediaPFM/data/haozhe/onevision/llava_data",
+                "--image_folder", "/mediaPFM/data/haozhe/onevision/llava_data/geo3k/",
+                "--video_folder", "/mediaPFM/data/haozhe/onevision/llava_video",
+                "--mm_tunable_parts", "mm_vision_tower,mm_mlp_adapter,mm_language_model",
+                "--mm_vision_tower_lr", "2e-6",
+                "--vision_tower", "google/siglip-so400m-patch14-384",
+                "--mm_projector_type", "mlp2x_gelu",
+                "--mm_vision_select_layer", "-2",
+                "--mm_use_im_start_end", "False",
+                "--mm_use_im_patch_token", "False",
+                "--group_by_modality_length", "True",
+                "--image_aspect_ratio", "anyres_max_9",
+                "--image_grid_pinpoints", "(1x1),...,(6x6)",
+                "--mm_patch_merge_type", "spatial_unpad",
+                "--bf16", "True",
+                "--run_name", "test",
+                "--output_dir", "experiments/test",
+                "--num_train_epochs", "1",
+                "--per_device_train_batch_size", "1",
+                "--per_device_eval_batch_size", "4",
+                "--gradient_accumulation_steps", "2",
+                "--evaluation_strategy", "no",
+                "--save_strategy", "steps",
+                "--save_steps", "1000",
+                "--save_total_limit", "1",
+                "--learning_rate", "1e-5",
+                "--weight_decay", "0.",
+                "--warmup_ratio", "0.03",
+                "--lr_scheduler_type", "cosine",
+                "--logging_steps", "1",
+                "--tf32", "True",
+                "--model_max_length", "32768",
+                "--gradient_checkpointing", "True",
+                "--dataloader_num_workers", "4",
+                "--lazy_preprocess", "True",
+                "--report_to", "wandb",
+                "--torch_compile", "True",
+                "--torch_compile_backend", "inductor",
+                "--dataloader_drop_last", "True",
+                "--frames_upbound", "32",
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}
+
+
+// {
+//     // Use IntelliSense to learn about possible attributes.
+//     // Hover to view descriptions of existing attributes.
+//     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+//     "version": "0.2.0",
+//     "configurations": [
+//         {
+//             "name": "Python: Current File",
+//             "type": "debugpy",
+//             "request": "launch",
+//             "program": "docs/LLaVA_OneVision_Tutorials.py",
+//             "console": "integratedTerminal",
+//             "env":{"CUDA_VISIBLE_DEVICES":"0",
+//                    "LD_PRELOAD": "/usr/lib/x86_64-linux-gnu/libffi.so.7",
+//                    "LD_LIBRARY_PATH": "/home/haozhe/miniconda3/envs/llava/lib"},
+//             "justMyCode": false,
+//             // "args": [
+//             //     "--run_dir_name", "test",
+//             //     // "--use_big_decoder"
+//             // ]
+//         }
+//     ]
+// }

add_dataset_name.py

@@ -0,0 +1,33 @@
+import json
+import os
+
+json_root = '/mediaPFM/data/haozhe/onevision/llava_instruct_old'
+save_root = '/mediaPFM/data/haozhe/onevision/llava_instruct'
+
+json_list = os.listdir(json_root)
+for json_name in json_list:
+    json_path = os.path.join(json_root, json_name)
+    if json_path.endswith(".jsonl"):
+        cur_data_dict = []
+        with open(json_path, "r") as json_file:
+            for line in json_file:
+                cur_data_dict.append(json.loads(line.strip()))
+    elif json_path.endswith(".json"):
+        with open(json_path, "r") as json_file:
+            cur_data_dict = json.load(json_file)
+    else:
+        raise ValueError(f"Unsupported file type: {json_path}")
+    
+    dataset_name = json_path.split('/')[-1].split('.')[0]
+    for data in cur_data_dict:
+        data['dataset_name'] = dataset_name
+
+    # save back
+    save_path = os.path.join(save_root, json_name)
+    with open(save_path, "w") as json_file:
+        if json_path.endswith(".jsonl"):
+            for data in cur_data_dict:
+                json_file.write(json.dumps(data) + "\n")
+        elif json_path.endswith(".json"):
+            json.dump(cur_data_dict, json_file, indent=4)
+    aa = 1