update org

Author: nbbb24

ecg_bench/organize_results.py

@@ -4,12 +4,55 @@
 from ecg_bench.config import get_args
 
 def extract_file_info(file):
-    parts = file.split('_')
-    rag_used = parts[-2] == 'True'
-    rag_k = int(parts[-1].split('.')[0]) if rag_used else None
-    is_seed = 'seed' in file
-    seed_num = int(file.split('/')[-1].split('_')[1]) if is_seed else None
-    return rag_used, rag_k, is_seed, seed_num
+    filename = file.split('/')[-1]
+    parts = filename.split('_')
+    
+    if filename.startswith('seed_'):
+        # seed_{seed}_{perturb}_{rag}_{retrieval_base}_{retrieved_information}_{rag_k}_{rag_prompt_mode}_{normalized_rag_feature}.json
+        seed_num = int(parts[1])
+        perturb = parts[2]
+        rag_used = parts[3] == 'True'
+        
+        if rag_used:
+            retrieval_base = parts[4]
+            retrieved_information = parts[5]
+            rag_k = int(parts[6])
+            rag_prompt_mode = parts[7]+parts[8]
+            normalized_rag_feature = parts[9].split('.')[0]
+        else:
+            retrieval_base = retrieved_information = rag_prompt_mode = normalized_rag_feature = None
+            rag_k = None
+            
+        is_seed = True
+    else:
+        # statistical_results_{perturb}_{rag}_{retrieval_base}_{retrieved_information}_{rag_k}_{rag_prompt_mode}_{normalized_rag_feature}.json
+        perturb = parts[2]
+        rag_used = parts[3] == 'True'
+        
+        if rag_used:
+            retrieval_base = parts[4]
+            retrieved_information = parts[5]
+            rag_k = int(parts[6])
+            rag_prompt_mode = parts[7]+parts[8]
+            normalized_rag_feature = parts[9].split('.')[0]
+        else:
+            retrieval_base = retrieved_information = rag_prompt_mode = normalized_rag_feature = None
+            rag_k = None
+            
+        is_seed = False
+        seed_num = None
+    
+    return {
+        'rag_used': rag_used,
+        'rag_k': rag_k,
+        'is_seed': is_seed,
+        'seed_num': seed_num,
+        'perturb': perturb,
+        'retrieval_base': retrieval_base,
+        'retrieved_information': retrieved_information,
+        'rag_prompt_mode': rag_prompt_mode,
+        'normalized_rag_feature': normalized_rag_feature
+    }
 
 def process_seed_data(data):
     averages = data['averages']
@@ -28,42 +71,52 @@ def collect_results(json_files):
     statistical_no_rag = {}
     individual_seeds_rag = defaultdict(dict)
     statistical_rag = {}
+    config_info_no_rag = None
+    config_info_rag = {}
 
     for file in json_files:
-        rag_used, rag_k, is_seed, seed_num = extract_file_info(file)
+        info = extract_file_info(file)
         with open(file, 'r') as f:
             data = json.load(f)
 
-        if is_seed:
+        if info['is_seed']:
             metrics = process_seed_data(data)
-            if rag_used:
-                individual_seeds_rag[rag_k][seed_num] = metrics
+            if info['rag_used']:
+                individual_seeds_rag[info['rag_k']][info['seed_num']] = metrics
+                config_info_rag[info['rag_k']] = info
             else:
-                individual_seeds_no_rag[seed_num] = metrics
+                individual_seeds_no_rag[info['seed_num']] = metrics
+                config_info_no_rag = info
         else:
-            if rag_used:
-                statistical_rag[rag_k] = data
+            if info['rag_used']:
+                statistical_rag[info['rag_k']] = data
+                config_info_rag[info['rag_k']] = info
             else:
                 statistical_no_rag = data
+                config_info_no_rag = info
 
     return (individual_seeds_no_rag, statistical_no_rag,
-            individual_seeds_rag, statistical_rag)
+            individual_seeds_rag, statistical_rag, config_info_no_rag, config_info_rag)
 
-def print_seed_results(title, seed_dict):
+def print_seed_results(title, seed_dict, config_info=None):
     if not seed_dict:
         return
     print(title)
+    if config_info:
+        print(f"  Config: perturb={config_info['perturb']}, retrieval_base={config_info['retrieval_base']}, retrieved_info={config_info['retrieved_information']}, prompt_mode={config_info['rag_prompt_mode']}, normalized={config_info['normalized_rag_feature']}")
     for seed in sorted(seed_dict.keys()):
         print(f"  Seed {seed}:")
         for metric in ['BLEU', 'METEOR', 'ROUGE', 'BERTSCORE', 'ACC']:
             value = seed_dict[seed][metric] * 100  # Scale to 0-100
             print(f"    {metric}: {value:.2f}")
     print('--------------------------------')
 
-def print_statistical_results(title, stats_dict):
+def print_statistical_results(title, stats_dict, config_info=None):
     if not stats_dict:
         return
     print(title)
+    if config_info:
+        print(f"  Config: perturb={config_info['perturb']}, retrieval_base={config_info['retrieval_base']}, retrieved_info={config_info['retrieved_information']}, prompt_mode={config_info['rag_prompt_mode']}, normalized={config_info['normalized_rag_feature']}")
     for metric in ['BLEU', 'METEOR', 'ROUGE', 'BERTSCORE', 'ACC']:
         value = (stats_dict['ROUGE']['rouge-l'] if metric == 'ROUGE' else
                  stats_dict['BERTSCORE']['hf-f1'] if metric == 'BERTSCORE' else
@@ -89,14 +142,15 @@ def main():
         return
 
     (individual_seeds_no_rag, statistical_no_rag,
-     individual_seeds_rag, statistical_rag) = collect_results(json_files)
+     individual_seeds_rag, statistical_rag, config_info_no_rag, config_info_rag) = collect_results(json_files)
 
-    print_seed_results("Individual Seed Results without RAG:", individual_seeds_no_rag)
-    print_statistical_results("Statistical Results without RAG:", statistical_no_rag)
+    print_seed_results("Individual Seed Results without RAG:", individual_seeds_no_rag, config_info_no_rag)
+    print_statistical_results("Statistical Results without RAG:", statistical_no_rag, config_info_no_rag)
 
     for k in sorted(individual_seeds_rag.keys()):
-        print_seed_results(f"Individual Seed Results with RAG k={k}:", individual_seeds_rag[k])
-        print_statistical_results(f"Statistical Results with RAG k={k}:", statistical_rag.get(k, {}))
+        config_info = config_info_rag.get(k)
+        print_seed_results(f"Individual Seed Results with RAG k={k}:", individual_seeds_rag[k], config_info)
+        print_statistical_results(f"Statistical Results with RAG k={k}:", statistical_rag.get(k, {}), config_info)
 
     print('================================================')
 

ecg_bench/scripts/org_results.sh

@@ -2,8 +2,14 @@
 
 # data=("ecg-qa_ptbxl_mapped_1250" "pretrain_mimic_mapped_1250" "ecg_instruct_45k_mapped_1250" "ecg_instruct_pulse_mapped_1250" "ecg-qa_mimic-iv-ecg_mapped_1250")
 data=("ecg_instruct_45k_mapped_1250")
+# retrieval_base="feature"
+# retrieved_information="combined"
+# rag_k=1
+# rag_prompt_mode="system_prompt"
+# normalized_rag_features=True
+
 checkpoints=(
-    "llama-3.2-3b-instruct_2_1_1024_0.0001_0.9_0.99_1e-08_500_0.01_True_None_None_None_True_combined_report_5_False"
+    'llama-3.2-3b-instruct_2_1_1024_0.0001_0.9_0.99_1e-08_500_0.01_True_None_None_None_True_combined_combined_1_system_prompt_True_False'
 )
 
 for d in "${data[@]}"; do