udates

Author: Haozhe Qi

llava/action/utils.py

@@ -270,7 +270,7 @@ def format_task_related_prompt(question, question_type, meta_data = None, perspe
 
     elif question_type == "direct_narration":
 
-        if learn_neighbor_actions == "prior" and meta_data and random.random() < 0.5:
+        if learn_neighbor_actions == "prior" and meta_data and random.random() < 0.1:
             ret = f"{perspective_prefix} {prev2_offset} seconds ago, you started an action {prev2_narration}. {prev1_offset} seconds ago, you started an action {prev1_narration}. What action are you currently performing? Give a short sentence such as 'move knife'. "            
         else:            
             ret = f"{perspective_prefix} What action are you performing? Give a short sentence such as 'move knife'."

llava/train/train.py

@@ -989,7 +989,8 @@ def __init__(self, data_path: str, tokenizer: transformers.PreTrainedTokenizer,
 
         from llava.action.generate_interval_pred import get_lookup_dict
 
-        self.train_triple_lookup = get_lookup_dict(os.path.join(self.EK100_anno_root, 'EPIC_100_train.csv'), self.eval_args.action_representation)
+        self.train_triple_lookup_official = get_lookup_dict(os.path.join(self.EK100_anno_root, 'EPIC_100_train.csv'), 'official_key')
+        self.train_triple_lookup_narration = get_lookup_dict(os.path.join(self.EK100_anno_root, 'EPIC_100_train.csv'), 'GT_random_narration')
 
         # Handle multiple JSON files specified in the data_path
         if "{" in data_path and "}" in data_path:
@@ -1282,7 +1283,12 @@ def _get_item(self, i) -> Dict[str, torch.Tensor]:
                     start_timestamp = round(float(self.list_data_dict[i]['start_timestamp']), 2)
                     end_timestamp = round(float(self.list_data_dict[i]['end_timestamp']), 2)
                     uid = f"{vid}_{start_timestamp}_{end_timestamp}"
-                    meta_data = self.train_triple_lookup.get(uid, None)
+                    # if True:
+                    #     meta_data = self.train_triple_lookup_narration.get(uid, None)
+                    if 'official_key' in sources[0]['question_type']:
+                        meta_data = self.train_triple_lookup_official.get(uid, None)
+                    elif 'GT_random_narration' in sources[0]['question_type']:
+                        meta_data = self.train_triple_lookup_narration.get(uid, None)
 
 
                 if 'EK100' not in video_file and 'EKframes' not in video_folder:

run_llmseval_clariden.sbatch

@@ -55,11 +55,11 @@ PYTHON_FILE="-m lmms_eval"
 PYTHON_ARGS=" \
     --model llava_vid \
     --model_args pretrained=experiments/LLaVA-Video-7B-Qwen2,conv_template=qwen_1_5,max_frames_num=64,mm_spatial_pool_mode=average \
-    --tasks videomme \
+    --tasks videomme,egoschema,nextqa \
     --batch_size 1 \
     --log_samples \
     --log_samples_suffix llava_vid \
-    --output_path ./logs/
+    --output_path ./benchmarks/
     --verbosity=DEBUG \
     "