Merge pull request #99 from openproblems-bio/jalil

all metrics integrated

Author: janursa

scripts/run_all.sh

@@ -1,12 +1,12 @@
 set -e
 
-datasets=('replogle'  'nakatake' 'adamson' 'norman'  'xaira_HEK293T' 'xaira_HCT116'  'parsebioscience' 'ibd' '300BCG') #'replogle' 'op' 'nakatake' 'adamson' 'norman'  'xaira_HEK293T' 'xaira_HCT116'  'parsebioscience' 'ibd' '300BCG') #
+datasets=('replogle' 'op') #'replogle' 'op' 'nakatake' 'adamson' 'norman'  'xaira_HEK293T' 'xaira_HCT116'  'parsebioscience' 'ibd' '300BCG') #
 
 run_local=false # set to true to run locally, false to run on AWS
 
 run_grn_inference=false
-run_grn_evaluation=true
-run_download=false
+run_grn_evaluation=false
+run_download=true
 
 
 for dataset in "${datasets[@]}"; do
@@ -56,7 +56,7 @@ for dataset in "${datasets[@]}"; do
         fi
 
         echo "Running GRN evaluation for dataset: $dataset"
-        # bash scripts/run_grn_evaluation.sh --dataset=$dataset --run_local=$run_local --build_images=false 
+        bash scripts/run_grn_evaluation.sh --dataset=$dataset --run_local=$run_local --build_images=false 
     fi
 
     if [ "$run_download" = true ]; then

scripts/run_grn_evaluation.sh

@@ -65,7 +65,7 @@ fi
 
 
 num_workers=10
-metric_ids="[regression, ws_distance, sem, tf_recovery, replica_consistency]" #
+metric_ids="[all_metrics]" # regression, ws_distance, sem, tf_recovery, replica_consistency
 RUN_ID="${DATASET}_evaluation"
 models_folder="${DATASET}/"
 apply_tf=true

src/api/comp_metric.yaml

@@ -59,6 +59,29 @@ arguments:
     direction: input
     default: ridge
     description: name of regression to use
+  - name: --ground_truth
+    type: file
+    direction: input
+    must_exist: false
+    required: false
+    example: resources_test/grn_benchmark/ground_truth/PBMC.csv 
+  - name: --ws_consensus
+    type: file
+    direction: input
+    must_exist: false
+    required: false
+    example: resources_test/grn_benchmark/prior/ws_consensus_norman.csv 
+  - name: --ws_distance_background
+    type: file
+    direction: input
+    must_exist: false
+    required: false
+    example: resources_test/grn_benchmark/prior/ws_distance_background_norman.csv
+  - name: --silent_missing_dependencies 
+    type: boolean
+    required: false
+    direction: input
+    default: true
 
 
 test_resources:

src/metrics/all_metrics/config.vsh.yaml

@@ -12,14 +12,33 @@ resources:
   - path: helper.py
   - path: /src/utils/util.py
     dest: util.py
+  - path: /src/metrics/regression/helper.py
+    dest: regression_helper.py
+  - path: /src/metrics/ws_distance/helper.py
+    dest: ws_helper.py
+  - path: /src/metrics/sem/helper.py
+    dest: sem_helper.py
+  - path: /src/metrics/tf_recovery/helper.py
+    dest: tf_recovery_helper.py
+  - path: /src/metrics/tf_binding/helper.py
+    dest: tf_binding_helper.py
+  - path: /src/metrics/replica_consistency/helper.py
+    dest: replica_consistency_helper.py
+  - path: /src/utils/dataset_config.py
+    dest: dataset_config.py
+  - path: /src/metrics/metrics_config.py
+    dest: metrics_config.py
+  
+  
+  
 
 engines:
   - type: docker
     image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-devel
     __merge__: /src/api/base_requirements.yaml
     setup:
       - type: python
-        packages: [ lightgbm==4.3.0, numpy==1.26.4 , tqdm_joblib==0.0.5]
+        packages: [ lightgbm==4.3.0, numpy==1.26.4 , tqdm_joblib==0.0.5, decoupler==2.1.1]
 
 
 runners:

src/metrics/all_metrics/helper.py

@@ -7,13 +7,39 @@
 import sys
 import os
 
+try:
+    from regression_helper import main as main_reg
+except:
+    from regression.helper import main as main_reg
+
+try:
+    from ws_helper import main as main_ws_distance
+except:
+    from ws_distance.helper import main as main_ws_distance
+
+try:
+    from sem_helper import main as main_sem
+except:
+    from sem.helper import main as main_sem
+
+
+try:
+    from tf_recovery_helper import main as main_tf_rec
+except:
+    from tf_recovery.helper import main as main_tf_rec
+
+
+try:
+    from tf_binding_helper import main as main_tf_binding
+except:
+    from tf_binding.helper import main as main_tf_binding
+
+
+try:
+    from replica_consistency_helper import main as main_replica_consistency
+except:
+    from replica_consistency.helper import main as main_replica_consistency
 
-from regression.helper import main as main_reg2
-from ws_distance.helper import main as main_ws_distance
-from sem.helper import main as main_sem
-from tf_recovery.helper import main as main_tf_rec
-from tf_binding.helper import main as main_tf_binding
-from replica_consistency.helper import main as main_replica_consistency
 from metrics_config import datasets_metrics
 
 
@@ -51,7 +77,7 @@ def replica_consistency_metric(par, dataset_id):
 def reg2_metric(par, dataset_id):
     if dataset_id in datasets_metrics:
         if 'regression' in datasets_metrics[dataset_id]:
-            output = main_reg2(par)
+            output = main_reg(par)
             return output
     return None
 

src/metrics/all_metrics/run.sh

@@ -1,6 +1,6 @@
 
 viash run src/metrics/all_metrics/config.vsh.yaml -- \
-    --prediction resources/grn_models/op/scprint.h5ad \
+    --prediction resources/results/op/op.scprint.scprint.prediction.h5ad \
     --evaluation_data resources/grn_benchmark/evaluation_data/op_bulk.h5ad \
     --score output/score.h5ad \
     --regulators_consensus resources/grn_benchmark/prior/regulators_consensus_op.json 

src/metrics/all_metrics/script.py

@@ -26,6 +26,8 @@
 
 try:
   sys.path.append(meta["resources_dir"])
+  from sem_helper import main as main_sem
+  print(main_sem)
 except:
     meta = {
       "all_dir":'src/metrics/',

src/metrics/replica_consistency/script.py

@@ -33,14 +33,6 @@
   method_id = ad.read_h5ad(par['prediction'], backed='r').uns['method_id']
   dataset_id = ad.read_h5ad(par['evaluation_data'], backed='r').uns['dataset_id']
 
-  try:
-    output = main(par)
-  except Exception as e:
-    print({'error': str(e)})
-
-    output = pd.DataFrame({
-        'key': ["None"],
-        'value': ["None"],
-    })
+  output = main(par)
 
   format_save_score(output, method_id, dataset_id, par['score'])

src/metrics/sem/config.vsh.yaml

@@ -21,7 +21,7 @@ engines:
     __merge__: /src/api/base_requirements.yaml
     setup:
       - type: python
-        packages: [ ]
+        packages: []
 runners:
   - type: executable
   - type: nextflow

src/metrics/sem/script.py

@@ -49,15 +49,9 @@
         par[key] = value
 
 if __name__ == "__main__":
-    try:
-        output = main_sem(par)
-    except Exception as e:
-        print(f"Error in SEM evaluation: {e}")
-        output = pd.DataFrame({
-            'key': ["None"],
-            'value': ["None"]
-        })
-
+    
+    output = main_sem(par)
+    
     dataset_id = ad.read_h5ad(par['evaluation_data'], backed='r').uns['dataset_id']
     method_id = ad.read_h5ad(par['prediction'], backed='r').uns['method_id']
     format_save_score(output, method_id, dataset_id, par['score'])