updates

Author: Haozhe Qi

llava/action/ek_eval.py

@@ -124,7 +124,7 @@ def get_args_parser():
                                    'random_narration_cut', 'top1_narration_cut', 'topk_narration_cut_key',
                                    'GT_key', 'GT_random_narration', 'GT_random_narration_cut', 'gpt_narration'])
     parser.add_argument('--n_narrations', default = -1, type = int)
-    parser.add_argument('--test_type', default = 'base', type = str, choices = ['caption', 'base', 'temporal_cot', 'temporal_cot_oracle', 'caption_then_answer', 'direct_narration'])
+    parser.add_argument('--test_type', default = 'base', type = str, choices = ['caption', 'base', 'temporal_cot_caption', 'temporal_cot_pseudo', 'temporal_cot_oracle', 'caption_then_answer', 'direct_narration'])
     parser.add_argument('--learn_neighbor_actions', type= str, default = "")
     parser.add_argument('--pseudo_folder', default = None, type = str)
     parser.add_argument('--output_dir', default = None, type = str)

llava/action/generate_interval_pred.py

@@ -130,7 +130,7 @@ def get_lookup_dict(ann_file, test_type = 'base', delta = 3, pseudo_folder = Non
     table = {}
 
     pseudo_dict = None
-    if test_type == 'temporal_cot':
+    if test_type == 'temporal_cot_pseudo':
         assert os.path.exists(pseudo_folder), f"Folder {pseudo_folder} does not exist"
         pseudo_dict = get_pseudo_dict(pseudo_folder)
 
@@ -159,11 +159,11 @@ def get_lookup_dict(ann_file, test_type = 'base', delta = 3, pseudo_folder = Non
                 uid2 = f"{id}_{round(start_times[i+1],2)}_{round(end_times[i+1],2)}"
                 uid3 = f"{id}_{round(start_times[i+2],2)}_{round(end_times[i+2],2)}"
 
-                if test_type == 'base' or test_type == 'temporal_cot_oracle':
+                if test_type == 'base' or test_type.startswith('temporal_cot') and test_type != 'temporal_cot_pseudo':
                     narration1 = sorted_narrations[i]
                     narration2 = sorted_narrations[i+1]
                     narration3 = sorted_narrations[i+2]
-                elif test_type == 'temporal_cot':
+                elif test_type == 'temporal_cot_pseudo':
                     narration1 = pseudo_dict[uid1]
                     narration2 = pseudo_dict[uid2]
                     narration3 = sorted_narrations[i+2]

llava/action/llava_inference.py

@@ -53,10 +53,12 @@ def llava_inference(
             question_type = "direct_narration"
         elif test_type == 'caption' or test_type == 'debug':
             question_type = "caption"
-        elif test_type == 'temporal_cot':
-            question_type = 'temporal_cot'
+        elif test_type == 'temporal_cot_pseudo':
+            question_type = 'temporal_cot_pseudo'
         elif test_type == 'temporal_cot_oracle':
-            question_type = 'temporal_cot_oracle'
+            question_type = 'temporal_cot_oracle'            
+        elif test_type == 'temporal_cot_caption':
+            question_type = 'temporal_cot_caption'
 
         if  test_type == 'caption_then_answer':        
             caption_answer = llava_inference([video_frames], 

llava/action/utils.py

@@ -250,14 +250,15 @@ def format_task_related_prompt(question, question_type, meta_data = None, perspe
             ret = prefix + suffix
         elif question_type.startswith("temporal_cot") and learn_neighbor_actions == "prior" and meta_data:
             # means it's test time
-            prefix = f"{perspective_prefix}\n"
-            assert isinstance(question, list)
-            suffix = ", ".join(question)           
-            suffix = f"{prev2_offset} seconds ago, you started an action {prev2_narration}. {prev1_offset} seconds ago, you started an action {prev1_narration}. What action are you currently performing? Here are the options of actions you can select:\n" + suffix 
-            ret = prefix + suffix 
-            rank0_print('debug')
-            rank0_print (meta_data)
-            rank0_print (ret)
+            if question_type == 'temporal_cot_caption':
+                ret = f"{perspective_prefix} {prev2_offset} seconds ago, you started an action {prev2_narration}. {prev1_offset} seconds ago, you started an action {prev1_narration}. Describe in details what you see from the video frames. You must talk in the first person perspective. Try to focus on what you are doing."
+                rank0_print(ret)
+            else:
+                prefix = f"{perspective_prefix}\n"
+                assert isinstance(question, list)
+                suffix = ", ".join(question)           
+                suffix = f"{prev2_offset} seconds ago, you started an action {prev2_narration}. {prev1_offset} seconds ago, you started an action {prev1_narration}. What action are you currently performing? Here are the options of actions you can select:\n" + suffix 
+                ret = prefix + suffix             
 
         else:
             action_rep_suffix = "Given multiple choices, format your answer briefly such as 'A. move knife'. "