multi choice training

Author: Ye Shaokai

action/ek_eval.py

@@ -18,7 +18,7 @@
 import json
 import logging
 from llava.utils import rank0_print
-
+from action.utils import generate_label_map
 
 def datetime2sec(str):
     hh, mm, ss = str.split(':')
@@ -453,8 +453,7 @@ def __getitem__(self, i):
 
         # randomly sample topk actions from valid gts
 
-        wrong_answer_indices = np.random.choice(len(self.valid_gts), size = self.eval_args.topk_predictions, replace = False)
-        
+        wrong_answer_indices = np.random.choice(len(self.valid_gts), size = self.eval_args.topk_predictions, replace = False)        
         wrong_answers = [self.valid_gts[index] for index in wrong_answer_indices]
 
         for i in range(len(wrong_answers)):
@@ -501,37 +500,6 @@ def get_downstream_dataset(transform, crop_size, eval_args, subset='train', labe
         assert ValueError("subset should be either 'train' or 'val'")
 
 
-def generate_label_map(eval_args):
-    print("Preprocess ek100 action label space")
-    vn_list = []
-    mapping_vn2narration = {}
-    anno_root = Path(eval_args.val_metadata).parent
-    for f in [      
-        anno_root / 'EPIC_100_train.csv',
-        anno_root / 'EPIC_100_validation.csv',
-    ]:
-        csv_reader = csv.reader(open(f))
-        _ = next(csv_reader)  # skip the header
-        for row in csv_reader:
-            
-            vn = '{}:{}'.format(int(row[10]), int(row[12]))
-            narration = row[8]
-            if vn not in vn_list:
-                vn_list.append(vn)
-            if vn not in mapping_vn2narration:
-                mapping_vn2narration[vn] = [narration]
-            else:
-                mapping_vn2narration[vn].append(narration)
-            # mapping_vn2narration[vn] = [narration]
-    vn_list = sorted(vn_list)
-    print('# of action= {}'.format(len(vn_list)))
-    mapping_vn2act = {vn: i for i, vn in enumerate(vn_list)}
-
-    labels = [list(set(mapping_vn2narration[vn_list[i]])) for i in range(len(mapping_vn2act))]
-    print(labels[:5])    
-    return labels, mapping_vn2act
-
-
 def get_args_parser():
     parser = argparse.ArgumentParser(description='AVION finetune ek100 cls', add_help=False)
     parser.add_argument('--dataset', default='ek100_cls', type=str, choices=['ek100_mir'])
@@ -596,9 +564,7 @@ def get_topk_predictions(data, idx,  k):
         'option': {0: options}        
         }    
 
-    return mc_data, predictions, target
-    
-
+    return mc_data, predictions, target    
 
 def evaluate_on_EK100(eval_args, model= None, tokenizer= None, max_length= None, image_processor= None):
 
@@ -611,7 +577,7 @@ def evaluate_on_EK100(eval_args, model= None, tokenizer= None, max_length= None,
 
     crop_size = 336
 
-    labels, mapping_vn2act = generate_label_map(eval_args) 
+    labels, mapping_vn2act, _, _ = generate_label_map(Path(eval_args.val_metadata).parent) 
     val_dataset = get_downstream_dataset(
         val_transform_gpu, crop_size, eval_args, subset='val', label_mapping=mapping_vn2act,
         labels = labels

action/generate_description.py

@@ -1,84 +1,46 @@
 import json
 import csv
 import os
+import argparse
+from action.utils import generate_label_map, MultiChoiceGenerator
+from pathlib import Path
+
+
+GEN_TYPES = ['naive', 'random_mc', 'avion_mc']
 
 def datetime2sec(str):
     hh, mm, ss = str.split(':')
     return int(hh) * 3600 + int(mm) * 60 + float(ss)
 
-def generate_label_map(dataset):
-    if dataset == 'ek100_cls':
-        print("Preprocess ek100 action label space")
-        vn_list = []
-        mapping_vn2narration = {}
-        verb_ids = {}
-        noun_ids = {}
-        for f in [
-            '/data/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv',
-            '/data/shaokai/epic-kitchens-100-annotations/EPIC_100_validation.csv',
-        ]:
-            csv_reader = csv.reader(open(f))
-            _ = next(csv_reader)  # skip the header
-            for row in csv_reader:
-                vn = '{}:{}'.format(int(row[10]), int(row[12]))
-                narration = row[8]
-                if row[10] not in verb_ids.keys():
-                    verb_ids[row[10]] = row[9]
-                if row[12] not in noun_ids.keys():
-                    noun_ids[row[12]] = row[11]
-                if vn not in vn_list:
-                    vn_list.append(vn)
-                if vn not in mapping_vn2narration:
-                    mapping_vn2narration[vn] = [narration]
-                else:
-                    mapping_vn2narration[vn].append(narration)
-                # mapping_vn2narration[vn] = [narration]
-        vn_list = sorted(vn_list)
-        print('# of action= {}'.format(len(vn_list)))
-        mapping_vn2act = {vn: i for i, vn in enumerate(vn_list)}
-        labels = [list(set(mapping_vn2narration[vn_list[i]])) for i in range(len(mapping_vn2act))]
-        print(labels[:5])
-    elif dataset == 'charades_ego':
-        print("=> preprocessing charades_ego action label space")
-        vn_list = []
-        labels = []
-        with open('datasets/CharadesEgo/CharadesEgo/Charades_v1_classes.txt') as f:
-            csv_reader = csv.reader(f)
-            for row in csv_reader:
-                vn = row[0][:4]
-                vn_list.append(vn)
-                narration = row[0][5:]
-                labels.append(narration)
-        mapping_vn2act = {vn: i for i, vn in enumerate(vn_list)}
-        print(labels[:5])
-    elif dataset == 'egtea':
-        print("=> preprocessing egtea action label space")
-        labels = []
-        with open('datasets/EGTEA/action_idx.txt') as f:
-            for row in f:
-                row = row.strip()
-                narration = ' '.join(row.split(' ')[:-1])
-                labels.append(narration.replace('_', ' ').lower())
-                # labels.append(narration)
-        mapping_vn2act = {label: i for i, label in enumerate(labels)}
-        print(len(labels), labels[:5])
-    else:
-        raise NotImplementedError
-    return labels, mapping_vn2act, verb_ids, noun_ids
-
-
-def parse_train_ann(ann_file, verb_ids, noun_ids):
+def generate_train_ann(ann_file, verb_ids, noun_ids, gen_type = 'naive'):
+    assert gen_type in GEN_TYPES
     # epic kitchen uses csv
     csv_reader = csv.reader(open(ann_file))
     _ = next(csv_reader)
     ret = []
+    ann_root = Path(ann_file).parent
+    if gen_type == "random_mc":
+        mc_generator = MultiChoiceGenerator(ann_root)
+
     for row in csv_reader:
-        # start_frame, end_frame = row[6], row[7]
         start_timestamp, end_timestamp = datetime2sec(row[4]), datetime2sec(row[5])
-        narration = f'{verb_ids[row[10]]} {noun_ids[row[12]]}'
+        
         pid, vid = row[1:3]
-        vid_path = '{}-{}'.format(pid, vid)        
-        conversation = generate_naive_conversation(narration)
+        vid_path = '{}-{}'.format(pid, vid)
+
+        if gen_type == 'naive':
+            # here we directly use the names
+            verb_noun = f'{verb_ids[row[10]]} {noun_ids[row[12]]}'
+            conversation = generate_naive_conversation(verb_noun)
+        elif gen_type == "random_mc":
+            # here we use the index
+            vn_str = f'{row[10]}:{row[12]}'
+            mc_data = mc_generator.generate_multi_choice(vn_str, 5)
+            options = mc_data['option'][0]
+            gt_answer_letter = mc_data['gt_answer_letter'][0]
+            gt_answer_name = mc_data['gt_answer_name'][0]
+            conversation = generate_random_mc_conversation(options, gt_answer_letter, gt_answer_name )
+
         data = {'video': vid_path,
                 'conversations': conversation,
                 'id': vid_path,
@@ -92,19 +54,35 @@ def parse_train_ann(ann_file, verb_ids, noun_ids):
         ret.append(data)
     return ret
 
-def generate_naive_conversation(narration):
+def generate_naive_conversation(vn_str:str):
     # in this version, we do not care about diversifying the questions
     return [
         {"from": "human", "value": "<image>\n the video is taken from egocentric view. What action is the person performing? Hint: provide your answer in verb-noun pair. "},
-        {"from": "gpt", "value": f"{narration}"}    
+        {"from": "gpt", "value": f"{vn_str}"}    
     ]
 
-def main():
+def generate_random_mc_conversation(options:list[str], gt_answer_letter, gt_answer_name):
+    return [
+        {"from": "human", "value": f"<image>\n the video is taken from egocentric view. What action is the person performing? Please select the letter for the right answer {options}"},
+        {"from": "gpt", "value": f"{gt_answer_letter}. {gt_answer_name}"} 
+    ]
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description="For generating VQA for EPIC-KITCHEN")
+    parser.add_argument('--train_metadata', default='/data/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv', type=str)
+    parser.add_argument('--out_folder', default = '/data/shaokai/EK100_in_LLAVA/', type = str)
+    return parser.parse_args()
+
+def main():    
+    args = get_args()    
+    ann_file = args.train_metadata
+    inst_train_folder = args.out_folder
+    print (ann_file)
+    anno_path = Path(ann_file).parent
+    labels, mapping_vn2act, verb_ids, noun_ids = generate_label_map(anno_path)
+    conv_lst = generate_train_ann(ann_file, verb_ids, noun_ids, gen_type = 'random_mc')
 
-    ann_file = "/data/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv"
-    labels, mapping_vn2act, verb_ids, noun_ids = generate_label_map('ek100_cls')
-    conv_lst = parse_train_ann(ann_file, verb_ids, noun_ids)
-    inst_train_folder = '/data/shaokai/EK100_in_LLAVA/'
     os.makedirs(inst_train_folder, exist_ok=True)
 
     # save it to a jsonl
@@ -113,6 +91,5 @@ def main():
             f.write(json.dumps(conv) + '\n')
 
 
-
 if __name__ == "__main__":
     main()

action/utils.py

@@ -0,0 +1,101 @@
+import csv 
+import numpy as np
+import random
+import os
+
+def generate_label_map(anno_root):
+    print("Preprocess ek100 action label space")
+    vn_list = []
+    mapping_vn2narration = {}
+    # from id to name
+    verb_maps = {}
+    noun_maps = {}
+    for f in [      
+        os.path.join(anno_root,'EPIC_100_train.csv'),
+        os.path.join(anno_root, 'EPIC_100_validation.csv'),
+    ]:
+        csv_reader = csv.reader(open(f))
+        _ = next(csv_reader)  # skip the header
+        for row in csv_reader:
+            
+            vn = '{}:{}'.format(int(row[10]), int(row[12]))
+            narration = row[8]
+            if row[10] not in verb_maps.keys():
+                verb_maps[row[10]] = row[9]
+            if row[12] not in noun_maps.keys():
+                noun_maps[row[12]] = row[11]
+
+            if vn not in vn_list:
+                vn_list.append(vn)
+            if vn not in mapping_vn2narration:
+                mapping_vn2narration[vn] = [narration]
+            else:
+                mapping_vn2narration[vn].append(narration)
+            # mapping_vn2narration[vn] = [narration]
+    vn_list = sorted(vn_list)
+    print('# of action= {}'.format(len(vn_list)))
+    mapping_vn2act = {vn: i for i, vn in enumerate(vn_list)}
+
+    labels = [list(set(mapping_vn2narration[vn_list[i]])) for i in range(len(mapping_vn2act))]
+    return labels, mapping_vn2act, verb_maps, noun_maps
+
+
+
+class MultiChoiceGenerator:
+    """
+    Generating multi choice
+    """
+    def __init__(self, ann_root):
+        self.ann_root = ann_root
+        _, self.mapping_vn2act, self.verb_maps, self.noun_maps = generate_label_map(ann_root)
+
+    def generate_multi_choice(self, gt_vn, k):
+        """
+        Generate k multiple choices from gt_vn pairs
+
+        randomly pick 1 letter for gt_vn
+        randomly pick k-1 letters from vn_list
+
+        """        
+
+        # let v_id and n_id be string type
+        gt_v_id, gt_n_id = gt_vn.split(':')    
+        assert isinstance(gt_v_id, str) and isinstance(gt_n_id, str)
+        gt_v_name, gt_n_name = self.verb_maps[gt_v_id], self.noun_maps[gt_n_id]
+
+        # letters as A, B, C, D, .. Note we maximally support 26 letters
+        letters = [chr(65+i) for i in range(26)][:k]
+        options = list(range(26))[:k]
+        vn_list = list(self.mapping_vn2act.keys())
+        action_list = [f"{self.verb_maps[e.split(':')[0]]} {self.noun_maps[e.split(':')[1]]}" for e in vn_list]
+        wrong_answers = np.random.choice(action_list, size = k-1, replace = False)
+        gt_answer = f'{gt_v_name} {gt_n_name}'
+
+        answers = [gt_answer] + list(wrong_answers)
+        random.shuffle(answers)
+
+        options = []
+        for answer, letter in zip(answers, letters):
+            options.append(f'{letter}. {answer}')
+
+        gt_letter = letters[answers.index(gt_answer)]
+        data = {
+                'question': {0: 'the video is an egocentric view of a person. What is the person doing? Pick the the letter that has the correct answer'},
+                'option': {0: options},
+                # the correct letter in mc
+                # for inspecting
+                'gt_answer_letter': {0: gt_letter},
+                'gt_answer_name': {0: gt_answer}
+            }
+        
+        return data
+    
+
+if __name__ == '__main__':
+
+    anno_root = "/storage-rcp-pure/upmwmathis_scratch/shaokai/epic-kitchens-100-annotations/"
+    generator = MultiChoiceGenerator(anno_root)
+    
+    print (generator.generate_multi_choice('3:3',5))
+
+    pass

llava/model/builder.py

@@ -216,9 +216,9 @@ def load_from_hf(repo_id, filename, subfolder=None):
                 else:
                     from llava.model.language_model.llava_qwen import LlavaQwenConfig
 
-                    if overwrite_config is not None:
+                    #if overwrite_config is not None:
+                    if True:
                         llava_cfg = LlavaQwenConfig.from_pretrained(model_path)
-                        rank0_print(f"Overwriting config with {overwrite_config}")
                         for k, v in overwrite_config.items():
                             setattr(llava_cfg, k, v)
                         model = LlavaQwenForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, attn_implementation=attn_implementation, config=llava_cfg, **kwargs)

llava/model/language_model/llava_qwen.py

@@ -47,9 +47,7 @@ class LlavaQwenForCausalLM(Qwen2ForCausalLM, LlavaMetaForCausalLM):
     config_class = LlavaQwenConfig
 
     def __init__(self, config):
-        # super(Qwen2ForCausalLM, self).__init__(config)
-        print ('what does config look like')
-        print (config)
+        # super(Qwen2ForCausalLM, self).__init__(config)      
         Qwen2ForCausalLM.__init__(self, config)
 
         config.model_type = "llava_qwen"

llava/train/train.py

@@ -1733,7 +1733,9 @@ def make_inputs_require_grad(module, input, output):
 
 
     if list(pathlib.Path(training_args.output_dir).glob("checkpoint-*")):
-        trainer.train(resume_from_checkpoint=True)
+        #trainer.train(resume_from_checkpoint=True)
+        # for debug purpose, let's not resume
+        trainer.train()
     else:
         trainer.train()
     trainer.save_state()

scripts/train/EK100.yaml

@@ -1,3 +1,3 @@
 datasets:
-  - json_path: /data/shaokai/EK100_in_LLAVA/train_convs_narration.jsonl
+  - json_path: /data/shaokai/EK100_avion_mc/train_convs_narration.jsonl
     sampling_strategy: all

shaokai_generate_train.sh

@@ -0,0 +1,8 @@
+# python3 action/generate_description.py  \
+#     --train_metadata /data/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv \
+#     --out_folder /data/shaokai/EK100_avion_mc/ \
+#  > train_gen.out 2>&1
+
+python3 action/generate_description.py \
+   --train_metadata /storage-rcp-pure/upmwmathis_scratch/shaokai/epic-kitchens-100-annotations/EPIC_100_train.csv \
+   --out_folder /storage-rcp-pure/upmwmathis_scratch/shaokai/EK100_avion_mc #  > train_gen.out 2>&1