updates

Author: Ye Shaokai

action/ek_eval.py

@@ -19,6 +19,7 @@
 import logging
 from llava.utils import rank0_print
 from action.utils import generate_label_map, MultiChoiceGenerator, match_answer, parse_avion_predictions
+from action.prediction_analysis import PredictionAnalysis
 import copy
 from collections import Counter 
 import torch.distributed as dist
@@ -224,6 +225,10 @@ def get_raw_item(
                                   fast_rcc=fast_rcc,
                                   rcc_params=rcc_params,
                                   jitter=is_training)
+            time_meta['start_second'] = start_second
+            time_meta['end_second'] = end_second
+            time_meta['fps'] = fps
+            time_meta['vid_path'] = vid_path
             return frames, '{}:{}'.format(verb, noun), time_meta
         else:
             raise NotImplementedError
@@ -271,10 +276,13 @@ def __init__(
         self.verb_maps = verb_maps
         self.noun_maps = noun_maps
         self.vn_list = list(self.label_mapping.keys())        
+
         self.labels = labels
         self.topk_predictions = topk_predictions
         self.ann_root = Path(metadata).parent
         self.mc_generator = MultiChoiceGenerator(self.ann_root)
+        self.rank = dist.get_rank()
+        self.prediction_analysis = PredictionAnalysis(f'prediction_analysis_buf_rank{self.rank}.json')
 
     def __getitem__(self, i):
         frames, label, time_meta = self.get_raw_item(
@@ -299,8 +307,6 @@ def __getitem__(self, i):
 
         data = self.mc_generator.generate_multi_choice(label, self.topk_predictions)
 
-        dataset_size = len(self.samples)
-
         return frames, data, time_meta, i
 
 
@@ -330,10 +336,11 @@ def get_args_parser():
     # llm size is type of string and can only be '7b' or '5b' etc.
     parser.add_argument('--pretrained_name', default = '', type = str, help ='the name in huggingface')
     parser.add_argument('--llava_num_frames', default=16, type=int, help='number of frames for llava')
-    ## avaion refinement 
+    ## avion refinement 
     parser.add_argument('--action_predictions', default=None, type=str, help='path to action predictions')
     parser.add_argument('--topk_predictions', default = 5, type =int)
     parser.add_argument('--llava_checkpoint', default = None, type = str)
+    parser.add_argument('--early_stop', default = None, type = int)
 
     return parser
 
@@ -438,7 +445,7 @@ def ensemble_llava_evaluation(
     rank0_print ('inspecting the counter', counter)
     rank0_print ('most common', counter.most_common(1)[0][0])
 
-    return match_answer(counter.most_common(1)[0][0], gt_name)
+    return match_answer(counter.most_common(1)[0][0], gt_name), counter.most_common(1)[0][0]
 
 
 
@@ -497,7 +504,7 @@ def evaluate_on_EK100(eval_args,
     print ('pretrained', pretrained)
 
     # so we know it's evaluation during training
-    finish_early = model is not None
+    finish_early = False #model is not None
 
     if model is None:
         if args.llava_checkpoint is not None:
@@ -508,26 +515,40 @@ def evaluate_on_EK100(eval_args,
         with open(eval_args.action_predictions, 'r') as f:
             predictions = json.load(f)        
 
-    avaion_correct = torch.tensor(0, device='cuda')
-    running_corrects = torch.tensor(0, device='cuda')
-    total_samples = torch.tensor(0, device='cuda')
+    device = torch.device(f'cuda:{rank}') 
+
+    global_avion_correct = torch.tensor(0.0, device=device)
+    global_running_corrects = torch.tensor(0.0, device=device)
+    global_total_samples = torch.tensor(0.0, device=device)
+
 
     for idx, (frames, mc_data, time_meta, global_index) in tqdm(enumerate(val_dataloader)):        
+
+        global_index = global_index.item()
+
         gt_name = mc_data['gt_answer_name'][0][0]
+        local_avion_correct = torch.tensor(0.0, device=device)
+        local_running_corrects = torch.tensor(0.0, device=device)
+        local_total_samples = torch.tensor(0.0, device=device)
 
         if eval_args.action_predictions:
-            mc_data = get_topk_predictions(predictions, global_index.item(), eval_args.topk_predictions)
+            mc_data = get_topk_predictions(predictions, global_index, eval_args.topk_predictions)
             avion_pred = mc_data['avion_pred']
             if gt_name == avion_pred:
-                avaion_correct+=1
+                local_avion_correct.add_(1)
+                global_avion_correct.add_(1)
 
         # we don't want to evaluate the whole thing
         # let's evaluate 1000 samples to get the complete picture       
         if finish_early and idx> (1000 / dist.get_world_size()):
             break                     
 
+        if eval_args.early_stop and idx > eval_args.early_stop:
+            break
+
         # Update running corrects and total samples
-        running_corrects += ensemble_llava_evaluation(
+        
+        llava_correct, llava_pred = ensemble_llava_evaluation(
                                                       eval_args.pretrained_name,
                                                       gt_name,
                                                       frames, 
@@ -541,33 +562,69 @@ def evaluate_on_EK100(eval_args,
                                                       ensemble_k = 1,
                                                       time_meta = time_meta,
                                                       is_test = not finish_early)
+
+        # log the predictions into prediciton analysis
+
+        # val_dataset.prediction_analysis.log(global_index,
+        #                                     llava_pred,
+        #                                     gt_name,
+        #                                     predictions[str(global_index)],
+        #                                     time_meta['start_second'].item(),
+        #                                     time_meta['end_second'].item(),
+        #                                     time_meta['vid_path'],
+        #                                     dataset_name = 'EK100')
+
+        
+
+
+        local_running_corrects.add_(llava_correct)
+        global_running_corrects.add_(llava_correct)
 
-        total_samples += 1
+        local_total_samples.add_(1)
+        global_total_samples.add_(1)
+
+        logger.info(f'Process {dist.get_rank()} - local_total_samples: {local_total_samples:.4f}')
+
+        logger.info(f'Process {dist.get_rank()} - loca_llava_correct: {llava_correct:.4f}')
+
+        logger.info(f'Process {dist.get_rank()} - local_running_corrects: {local_running_corrects:.4f}')
+
 
         # Calculate and log running mean accuracy
-        running_accuracy = running_corrects / total_samples
+        # dist.barrier()
+        # dist.all_reduce(local_running_corrects, op=dist.ReduceOp.SUM)
+        # dist.all_reduce(local_total_samples, op=dist.ReduceOp.SUM)
+        # if eval_args.action_predictions:
+        #     dist.all_reduce(local_avion_correct, op=dist.ReduceOp.SUM)
+        # dist.barrier()
+        # # Calculate global accuracy after reduction
+        # local_running_accuracy = local_running_corrects.item() / local_total_samples.item()
+        # local_avion_accuracy = local_avion_correct.item() / local_total_samples.item()
+
+        # logger.info(f'Process {dist.get_rank()} - Running accuracy: {local_running_accuracy:.4f}')
+        # logger.info(f'Process {dist.get_rank()} - AvionRunning accuracy: {local_avion_accuracy:.4f}')
 
-        logger.info(f'Process {dist.get_rank()} - Running accuracy: {running_accuracy:.4f}')
-        if eval_args.action_predictions:
-            avaion_accuracy = avaion_correct / total_samples
+    
 
     dist.barrier()
-    dist.all_reduce(running_corrects, op=dist.ReduceOp.SUM)
-    dist.all_reduce(total_samples, op=dist.ReduceOp.SUM)
+    dist.all_reduce(global_running_corrects, op=dist.ReduceOp.SUM)
+    dist.all_reduce(global_total_samples, op=dist.ReduceOp.SUM)
     if eval_args.action_predictions:
-        dist.all_reduce(avaion_correct, op=dist.ReduceOp.SUM)
+        dist.all_reduce(global_avion_correct, op=dist.ReduceOp.SUM)
 
     # Calculate global accuracy after reduction
-    global_accuracy = running_corrects.item() / total_samples.item()
+    global_accuracy = global_running_corrects.item() / global_total_samples.item()
     if eval_args.action_predictions:
-        global_avaion_accuracy = avaion_correct.item() / total_samples.item()
+        global_avion_accuracy = global_avion_correct.item() / global_total_samples.item()
 
     # Ensure only the main process (rank 0) prints the final result
     if dist.get_rank() == 0:
         if eval_args.action_predictions:
-            logger.info(f'Global Avaion Accuracy: {global_avaion_accuracy:.4f}')
+            logger.info(f'Global Avion Accuracy: {global_avion_accuracy:.4f}')
         logger.info(f'Final Global Accuracy: {global_accuracy:.4f}')
 
+    #val_dataset.prediction_analysis.save()
+    
     return global_accuracy
 
 

action/prediction_analysis.py

@@ -0,0 +1,101 @@
+import json
+import glob
+
+class PredictionAnalysis:
+    """
+    We save data that can be used for ad-hoc analysis
+
+    We want to save the following:
+
+    # saving global index to make distributed code work better
+    {global_index: {
+        llava_pred: pred_name,
+        gt_name: pred_name,
+        avion_preds: avion_predictions,
+        # to locate the video clip
+        dataset_name: '',
+        start_second: '',    
+        end_second: '',
+        vid_path: ''
+    }
+    """
+    def __init__(self, save_path):
+        self.save_path = save_path
+        self.data = {}
+    def log(self, 
+            global_index,
+            llava_pred,
+            gt_name,
+            avion_preds,
+            start_second,
+            end_second,
+            vid_path,
+            dataset_name = 'EK100',
+            ):
+        self.data[global_index] = {
+            'llava_pred': llava_pred,
+            'gt_name': gt_name,
+            'avion_preds': avion_preds,
+            'dataset_name' : dataset_name,
+            'start_second' : start_second,
+            'end_second': end_second,
+            'vid_path': vid_path
+        }
+
+        # print ('check what is here')
+        # print (self.data[global_index])
+
+    def save(self):
+        with open(self.save_path, 'w') as f:
+            json.dump(self.data, f, indent = 4)
+
+
+class Analysis:
+    """
+
+    This same code should be applied to the training too.
+
+    collect all the wrong top-1 prediction from avion
+    collect all the wrong top-1 prediction from llava
+
+    Determine percentage of wrong llava prediction that has wrong verb only
+    Determine percentage of wrong llava prediction that has wrong noun only
+    Determine percentage of wrong llava prediciton that has both verb and noun wrong
+    Determine percentage of wrong llava prediction that was wrong because the answer not in the top k
+    """
+    pass
+
+    def __init__(self, prefix):
+
+        files = glob.glob(prefix + '*')
+
+        self.data = {}
+
+        for file in files:
+            print ('loading pred checkpoint from: ', file)
+            with open(file, 'r') as f:
+                _data = json.load(f)
+                self.data.update(_data)
+
+        # add some assertion for number of keys in the data
+
+    def wrong_verb(self):
+
+        N = len(self.data)
+
+        wrong_verb_collections = []
+        wrong_noun_collections = []
+        wrong_verb_noun_collections = []
+
+        wrong_llava_collections = []
+        wrong_avion_collections = []
+
+        indices = sorted(self.data.keys())
+
+        for index in indices:
+            items = self.data[index]
+        
+
+
+if __name__ == '__main__':
+    pass

scripts/train/EK100_avion_mc_top10.yaml

@@ -0,0 +1,3 @@
+datasets:
+  - json_path: /data/shaokai/EK100_inst_train/avion_mc_top10/train_convs_narration.jsonl
+    sampling_strategy: all

shaokai_generate_train.sh

@@ -1,12 +1,14 @@
 # python3 action/generate_description.py  \
 #     --train_metadata /data/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv \
 #     --out_folder /data/shaokai/EK100_avion_mc/ \
-#  > train_gen.out 2>&1
+#     --gen_type avion_mc \
+#     --n_options 10 \
+#   > train_gen.out 2>&1
 
 python3 action/generate_description.py \
    --train_metadata /storage-rcp-pure/upmwmathis_scratch/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv \
    --out_folder /storage-rcp-pure/upmwmathis_scratch/shaokai/EK100_inst_train \
    --avion_train_predictions /storage-rcp-pure/upmwmathis_scratch/shaokai/avion_predictions_train.json \
    --gen_type avion_mc \
-   --n_options 3
+   --n_options 10
 

shaokai_llavavideo_train_avion_mc_7b_top10_5epochs.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Export environment variables
+export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
+export OMP_NUM_THREADS="8"
+export NCCL_IB_DISABLE="0"
+export NCCL_IB_GID_INDEX="3"
+export NCCL_SOCKET_IFNAME="eth0"
+export NCCL_DEBUG="INFO"
+export ACCELERATE_CPU_AFFINITY="1"
+export WANDB_API_KEY="4474ec79de023b0c3ffb43588ab6163264f875db"
+export HF_HOME=/data/shaokai
+
+
+# Run the command using torchrun
+torchrun --nproc_per_node=8 \
+         --nnodes=1 \
+         --node_rank=0 \
+         --master_addr=127.0.0.1 \
+         --master_port=29500 \
+         llava/train/train_mem.py \
+         --deepspeed scripts/zero3.json \
+         --model_name_or_path lmms-lab/LLaVA-Video-7B-Qwen2 \
+         --version qwen_1_5 \
+         --data_path scripts/train/EK100_avion_mc_top10.yaml \
+         --video_folder /data/shaokai/\
+         --mm_tunable_parts mm_vision_tower,mm_mlp_adapter,mm_language_model \
+         --mm_vision_tower_lr 2e-6 \
+         --vision_tower google/siglip-so400m-patch14-384 \
+         --mm_projector_type mlp2x_gelu \
+         --mm_vision_select_layer -2 \
+         --mm_use_im_start_end False \
+         --mm_use_im_patch_token False \
+         --group_by_modality_length True \
+         --image_aspect_ratio anyres_max_9 \
+         --image_grid_pinpoints "(1x1),...,(6x6)" \
+         --mm_patch_merge_type spatial_unpad \
+         --bf16 True \
+         --run_name shaokai_llava_video_7b_avion_mc_top10_5epochs \
+         --output_dir experiments/shaokai_llava_video_7b_avion_mc_top10_5epochs \
+         --num_train_epochs 5 \
+         --per_device_train_batch_size 2 \
+         --per_device_eval_batch_size 4 \
+         --gradient_accumulation_steps 2 \
+         --evaluation_strategy steps \
+         --eval_steps 2000\
+         --save_strategy steps \
+         --save_steps 1000 \
+         --learning_rate 1e-5 \
+         --weight_decay 0. \
+         --warmup_ratio 0.03 \
+         --lr_scheduler_type cosine \
+         --logging_steps 1 \
+         --tf32 True \
+         --model_max_length 32768 \
+         --gradient_checkpointing True \
+         --dataloader_num_workers 4 \
+         --lazy_preprocess True \
+         --report_to wandb \
+         --torch_compile True \
+         --torch_compile_backend inductor \
+         --dataloader_drop_last True \
+         --frames_upbound 32 \
+         --root /data/shaokai/EK100 \
+         --action_predictions /data/shaokai/avion_predictions_test.json \
+         --val_metadata /data/shaokai/epic-kitchens-100-annotations/EPIC_100_validation.csv \
+         --llava_num_frames 32 \
+         --clip_length 32 \
+         --topk_predictions 10 > train_llavavideo_kitchen_7b_avion_mc_32f_top10_5epochs.out 2>&1