Merge remote-tracking branch 'origin/main' into add_results_processing

Author: rcannood

CHANGELOG.md

@@ -122,7 +122,10 @@
 
 * Schema defined in `src\api` has been modified to include dataset specific parameters (PR #71).
 
-*  Disabled parallelization of `metrics/cms` + cms distributions are now kept in the output AnnData object(PR #83).
+* Disabled parallelization of `metrics/cms` + cms distributions are now kept in the output AnnData object (PR #83).
+
+* Add arguments for including/excluding methods and metrics in the benchmarking workflow (PR #100).
+
 
 ## BUG FIXES
 

_viash.yaml

@@ -50,6 +50,7 @@ authors:
     roles: [ "author", "maintainer" ]
     info: 
       github: LuLeom
+      orcid: 0009-0002-0742-8504
   - name: Givanna Putri
     roles: [ "author", "maintainer" ]
     info: 
@@ -64,10 +65,12 @@ authors:
     roles: [ "contributor" ]
     info: 
       github: KatrienQ
+      orcid: 0000-0001-5306-5615
   - name: Sofie Van Gassen
     roles: [ "contributor" ]
     info: 
       github: SofieVG
+      orcid: 0000-0002-7119-5330
 
 config_mods: |
   .runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }

scripts/run_benchmark/run_full_seqeracloud.sh

@@ -17,6 +17,7 @@ cat > /tmp/params.yaml << HERE
 input_states: s3://openproblems-data/resources/task_cyto_batch_integration/datasets/**/state.yaml
 rename_keys: 'input_censored_split1:output_censored_split1;input_censored_split2:output_censored_split2;input_unintegrated:output_unintegrated'
 output_state: "state.yaml"
+# settings: '{"methods_include": ["combat", "gaussnorm"]}'
 publish_dir: "$publish_dir"
 HERE
 

src/workflows/run_benchmark/config.vsh.yaml

@@ -48,17 +48,36 @@ argument_groups:
         default: task_info.yaml
   - name: Methods
     arguments:
-      - name: "--method_ids"
+      - name: "--methods_include"
         type: string
         multiple: true
-        description: A list of method ids to run. If not specified, all methods will be run.
+        description: |
+          A list of method ids to include. If specified, only these methods will be run.
+      - name: "--methods_exclude"
+        type: string
+        multiple: true
+        description: |
+          A list of method ids to exclude. If specified, all methods except the ones listed will be run.
+  - name: Metrics
+    arguments:
+      - name: "--metrics_include"
+        type: string
+        multiple: true
+        description: |
+          A list of metric ids to include. If specified, only these metrics will be run.
+      - name: "--metrics_exclude"
+        type: string
+        multiple: true
+        description: |
+          A list of metric ids to exclude. If specified, all metrics except the ones listed will be run.
 
 resources:
   - type: nextflow_script
     path: main.nf
     entrypoint: run_wf
   - type: file
     path: /_viash.yaml
+  - path: /common/nextflow_helpers/helper.nf
 
 dependencies:
   - name: utils/extract_uns_metadata

src/workflows/run_benchmark/main.nf

@@ -1,3 +1,5 @@
+include { checkItemAllowed } from "${meta.resources_dir}/helper.nf"
+
 workflow auto {
   findStates(params, meta.config)
     | meta.workflow.run(
@@ -81,9 +83,15 @@ workflow run_wf {
 
       // run only non-control methods & filter by method_ids
       filter: { id, state, comp ->
-        def id_filter = !state.method_ids || state.method_ids.contains(comp.config.name)
+        def method_check = checkItemAllowed(
+          comp.config.name,
+          state.methods_include,
+          state.methods_exclude,
+          "methods_include",
+          "methods_exclude"
+        )
         def method_filter = comp.config.info.type == "method"
-        id_filter && method_filter
+        method_check && method_filter
       },
 
       // define a new 'id' by appending the method name to the dataset id
@@ -127,9 +135,15 @@ workflow run_wf {
 
       // run only control methods & filter by method_ids
       filter: { id, state, comp ->
-        def id_filter = !state.method_ids || state.method_ids.contains(comp.config.name)
+        def method_check = checkItemAllowed(
+          comp.config.name,
+          state.methods_include,
+          state.methods_exclude,
+          "methods_include",
+          "methods_exclude"
+        )
         def method_filter = comp.config.info.type == "control_method"
-        id_filter && method_filter
+        method_check && method_filter
       },
 
       // define a new 'id' by appending the method name to the dataset id
@@ -160,6 +174,18 @@ workflow run_wf {
       id: { id, state, comp ->
         id + "." + comp.config.name
       },
+      filter: { id, state, comp ->
+        // filter by metric_ids
+        def metric_check = checkItemAllowed(
+          comp.config.name,
+          state.metrics_include,
+          state.metrics_exclude,
+          "metrics_include",
+          "metrics_exclude"
+        )
+        // filter by method_id
+        metric_check
+      },
       // use 'fromState' to fetch the arguments the component requires from the overall state
       fromState: [
         input_unintegrated: "input_unintegrated",