Merge pull request #100 from openproblems-bio/jalil

grn evaluation pipeline is using dataset specific metrics

Author: janursa

docs/source/images/raw_scores_op.png

@@ -5,8 +5,8 @@ datasets=('replogle' 'op') #'replogle' 'op' 'nakatake' 'adamson' 'norman'  'xair
 run_local=false # set to true to run locally, false to run on AWS
 
 run_grn_inference=false
-run_grn_evaluation=false
-run_download=true
+run_grn_evaluation=true
+run_download=false
 
 
 for dataset in "${datasets[@]}"; do

scripts/run_all.sh

@@ -65,7 +65,6 @@ fi
 
 
 num_workers=10
-metric_ids="[all_metrics]" # regression, ws_distance, sem, tf_recovery, replica_consistency
 RUN_ID="${DATASET}_evaluation"
 models_folder="${DATASET}/"
 apply_tf=true
@@ -102,6 +101,11 @@ param_aws="s3://openproblems-data/resources/grn/results/params/${RUN_ID}_param_l
 > "$param_local"
 > "$param_file"
 
+# Generate and source config file
+echo "Generating dataset configuration..."
+python src/utils/config.py --output src/utils/dataset_config.env
+source src/utils/dataset_config.env
+
 if [ "$RUN_LOCAL" = true ]; then
   cat >> "$param_local" << HERE
 param_list:
@@ -112,6 +116,16 @@ append_entry() {
   local grn_name="$1"
   local prediction="$2"
   local dataset="$3"
+
+  # Get cell type and metrics from sourced env variables
+  cell_type_var="CELLTYPE_${dataset}"
+  metrics_var="METRICS_${dataset}"
+
+  cell_type="${!cell_type_var}"
+  metric_ids="[${!metrics_var}]"
+
+  echo ${dataset}  ${cell_type} ${metric_ids}
+
   if [[ "$dataset" =~ ^(norman|nakatake|adamson)$ ]]; then
     layer_='X_norm'
   else
@@ -127,14 +141,22 @@ append_entry() {
     apply_tf: ${apply_tf}
     reg_type: ${reg_type}
     layer: $layer_
-
+    
 HERE
   # Additional fields for specific datasets
   if [[ "$dataset" =~ ^(norman|replogle|adamson|xaira_HCT116|xaira_HEK293T)$ ]]; then
     cat >> "$param_local" << HERE
     ws_consensus: ${resources_dir}/grn_benchmark/prior/ws_consensus_${dataset}.csv
     ws_distance_background: ${resources_dir}/grn_benchmark/prior/ws_distance_background_${dataset}.csv
     evaluation_data_de: ${resources_dir}/grn_benchmark/evaluation_data/${dataset}_de.h5ad
+HERE
+  fi
+
+  if [[ "$dataset" != "nakatake" ]]; then
+    cat >> "$param_local" << HERE
+    ground_truth_unibind: ${resources_dir}/grn_benchmark/ground_truth/${cell_type}_unibind.csv
+    ground_truth_chipatlas: ${resources_dir}/grn_benchmark/ground_truth/${cell_type}_chipatlas.csv
+    ground_truth_remap: ${resources_dir}/grn_benchmark/ground_truth/${cell_type}_remap.csv
 HERE
   fi
 }

scripts/run_grn_evaluation.sh

@@ -59,12 +59,26 @@ arguments:
     direction: input
     default: ridge
     description: name of regression to use
-  - name: --ground_truth
+  - name: --ground_truth_unibind
     type: file
     direction: input
     must_exist: false
     required: false
     example: resources_test/grn_benchmark/ground_truth/PBMC.csv 
+  - name: --ground_truth_chipatlas
+    type: file
+    direction: input
+    must_exist: false
+    required: false
+    example: resources_test/grn_benchmark/ground_truth/PBMC.csv 
+  - name: --ground_truth_remap
+    type: file
+    direction: input
+    must_exist: false
+    required: false
+    example: resources_test/grn_benchmark/ground_truth/PBMC.csv 
+  
+
   - name: --ws_consensus
     type: file
     direction: input

src/api/comp_metric.yaml

@@ -24,10 +24,8 @@ resources:
     dest: tf_binding_helper.py
   - path: /src/metrics/replica_consistency/helper.py
     dest: replica_consistency_helper.py
-  - path: /src/utils/dataset_config.py
-    dest: dataset_config.py
-  - path: /src/metrics/metrics_config.py
-    dest: metrics_config.py
+  - path: /src/utils/config.py
+    dest: config.py
 
 
 

src/metrics/all_metrics/config.vsh.yaml

@@ -40,33 +40,33 @@
 except:
     from replica_consistency.helper import main as main_replica_consistency
 
-from metrics_config import datasets_metrics
+from config import DATASETS_METRICS
 
 
 def sem_metric(par, dataset_id):
-    if dataset_id in datasets_metrics:
-        if 'sem' in datasets_metrics[dataset_id]:
+    if dataset_id in DATASETS_METRICS:
+        if 'sem' in DATASETS_METRICS[dataset_id]:
             output = main_sem(par)
             return output
     return None
 
 def tf_rec_metric(par, dataset_id):
-    if dataset_id in datasets_metrics:
-        if 'tf_recovery' in datasets_metrics[dataset_id]:
+    if dataset_id in DATASETS_METRICS:
+        if 'tf_recovery' in DATASETS_METRICS[dataset_id]:
             output = main_tf_rec(par)
             return output
     return None
 
 def tf_binding_metric(par, dataset_id):
-    if dataset_id in datasets_metrics:
-        if 'tf_binding' in datasets_metrics[dataset_id]:
+    if dataset_id in DATASETS_METRICS:
+        if 'tf_binding' in DATASETS_METRICS[dataset_id]:
             output = main_tf_binding(par)
             return output
     return None
 
 def replica_consistency_metric(par, dataset_id):
-    if dataset_id in datasets_metrics:
-        if 'replica_consistency' in datasets_metrics[dataset_id]:
+    if dataset_id in DATASETS_METRICS:
+        if 'replica_consistency' in DATASETS_METRICS[dataset_id]:
             try:
                 output = main_replica_consistency(par)
             except:
@@ -75,15 +75,15 @@ def replica_consistency_metric(par, dataset_id):
     return None
 
 def reg2_metric(par, dataset_id):
-    if dataset_id in datasets_metrics:
-        if 'regression' in datasets_metrics[dataset_id]:
+    if dataset_id in DATASETS_METRICS:
+        if 'regression' in DATASETS_METRICS[dataset_id]:
             output = main_reg(par)
             return output
     return None
 
 def ws_distance_metric(par, dataset_id):
-    if dataset_id in datasets_metrics:
-        if 'ws_distance' in datasets_metrics[dataset_id]:
+    if dataset_id in DATASETS_METRICS:
+        if 'ws_distance' in DATASETS_METRICS[dataset_id]:
             _, output = main_ws_distance(par)
             return output
     return None

src/metrics/all_metrics/helper.py

@@ -22,7 +22,7 @@
 np.random.seed(seed)
 
 from util import read_prediction, manage_layer
-from dataset_config import DATASET_GROUPS
+from config import DATASET_GROUPS
 
 
 def encode_obs_cols(adata, cols):

src/metrics/experimental/anchor_regression/helper.py

@@ -25,7 +25,7 @@
 np.random.seed(seed)
 
 from util import read_prediction, manage_layer
-from dataset_config import DATASET_GROUPS
+from config import DATASET_GROUPS
 
 
 def encode_obs_cols(adata, cols):

src/metrics/experimental/regression_3/helper.py

@@ -24,7 +24,7 @@
 
 from helper import main
 from util import format_save_score, parse_args
-# from dataset_config import DATASET_GROUPS
+# from config import DATASET_GROUPS
 
 par = parse_args(par)
 

src/metrics/experimental/replica_consistency/script.py

@@ -32,7 +32,7 @@
 torch.use_deterministic_algorithms(True)
 
 from util import read_prediction, manage_layer
-from dataset_config import DATASET_GROUPS
+from config import DATASET_GROUPS
 from scipy.spatial.distance import cityblock