make it run for ibis

.github/workflows/ci.yml

@@ -35,6 +35,9 @@ jobs:
           - "singularity"
         test_name:
           - "test"
+          - "test_ibis"
+          - "test_ibis_with_preprocessing"
+          - "test_noise_eval"
         isMaster:
           - ${{ github.base_ref == 'master' }}
         # Exclude conda and singularity on dev

conf/dev.config

@@ -0,0 +1,7 @@
+params {
+    config_profile_name        = 'Development profile'
+    config_profile_description = 'Params needed during development'
+
+    // container
+    container_dev = "docker.io/mathysgrapotte/stimulus-py:dev"
+}

conf/modules.config

@@ -75,7 +75,8 @@ process {
         publishDir = [
             path: {"${params.outdir}/data"},
             mode: params.publish_dir_mode,
-            pattern: "*.csv"
+            pattern: "*.csv",
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
         ext.prefix = { "${meta.id}.stimulus" }
         ext.suffix = { "csv" }
@@ -89,43 +90,91 @@ process {
     // main config
     // ==============================================================================
 
-    withName: "STIMULUS_SPLIT_SPLIT" {
+    withName: "STIMULUS_SPLIT_TRANSFORM" {
         publishDir = [
-            path: { "${params.outdir}/configs" },
-            mode: params.publish_dir_mode
+            path: { "${params.outdir}/configs/${meta.id}" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        tag = { "${meta.id}" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
     }
 
     withName: "STIMULUS_SPLIT_DATA" {
         publishDir = [
             enabled: false
         ]
-        ext.prefix = { "${meta.id}-split-${meta2.split_id}" }
+        ext.prefix = { "${meta.id}-split-${meta.split_id}" }
+        tag = { "${meta.id} - split: ${meta.split_id}" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
     }
 
     withName: "STIMULUS_TRANSFORM_CSV" {
         publishDir = [
-            path: { "${params.outdir}/data" },
+            path: { "${params.outdir}/transformed_data/${meta.id}" },
             mode: params.publish_dir_mode,
-            when: params.save_data
+            when: params.save_data,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
-        ext.prefix = { "${meta.id}-split-${meta2.split_id}-trans-${meta2.transform_id}" }
+        ext.prefix = { "${meta.id}-split-${meta.split_id}-trans-${meta.transform_id}" }
+        tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id}" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
     }
 
     withName: "CHECK_MODEL" {
         ext.args = { [
             params.check_model_num_samples ? "-n ${params.check_model_num_samples}" : '',
             params.debug_mode ? "--debug_mode" : ''
         ].flatten().unique(false).join(' ').trim()}
+        tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id}" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
     }
 
     withName: "STIMULUS_TUNE" {
         ext.args = { [
             params.debug_mode ? "--debug_mode" : ''
         ].flatten().unique(false).join(' ').trim()}
         publishDir = [
-            path: { "${params.outdir}/tune_results/${meta.id}" },
-            mode: params.publish_dir_mode
+            path: { "${params.outdir}/tune_results/${meta.id}/${meta.split_id}/${meta.transform_id}" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+        ext.prefix = { "${meta.id}-split-${meta.split_id}-trans-${meta.transform_id}-rep-${meta.replicate}" }
+        tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id} - replicate: ${meta.replicate}" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
+    }
+
+    withName: "STIMULUS_PREDICT" {
+        ext.prefix = { "${meta.id}-split-${meta.split_id}-transf-${meta.transform_id}-rep-${meta.replicate}" }
+        tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id} - replicate: ${meta.replicate}" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
+    }
+
+    withName: "STIMULUS_COMPARE_TENSORS_COSINE" {
+        ext.args = { "--mode cosine_similarity" }
+        if (params.container_dev) {
+            container = params.container_dev
+        }
+    }
+
+    withName: "CONCAT_COSINE" {
+        ext.prefix = { "summary_cosine" }
+        publishDir = [
+            path: { "${params.outdir}/evaluation_results/" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 }

conf/test_ibis.config

@@ -0,0 +1,47 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run main.nf -profile test_ibis,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'Test ibis profile'
+    config_profile_description = 'Minimal ibis test dataset to check pipeline functions'
+
+    // Input data
+    data                 = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ibis_SP140/SP140_shade.stimulus.csv'
+    data_config          = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/data.yaml'
+    model                = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ConvBasic_withEfficientKAN.py'
+    model_config         = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ConvBasic_withEfficientKAN.yaml'
+
+    // output
+    save_data    = true
+}
+
+// Limit resources so that this can run on GitHub Actions
+process {
+    maxRetries = params.max_retries
+    errorStrategy = params.err_start
+
+    withLabel:process_low {
+        cpus   = { 1                    }
+        memory = { 4.GB * task.attempt  }
+        time   = { 10.m  * task.attempt }
+    }
+    withLabel:process_medium {
+        cpus   = { 2                    }
+        memory = { 6.GB * task.attempt  }
+        time   = { 30.m  * task.attempt }
+    }
+    withLabel:process_high {
+        cpus   = { 4                    }
+        memory = { 8.GB * task.attempt  }
+        time   = { 1.h  * task.attempt  }
+    }
+}

conf/test_ibis_with_preprocessing.config

@@ -0,0 +1,50 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run main.nf -profile test_ibis,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'Test ibis profile - with preprocessing'
+    config_profile_description = 'Minimal ibis test dataset (with preprocessing) to check pipeline functions'
+
+    // Input data
+    data                 = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ibis_SP140/SP140_ghts.peaks'
+    data_config          = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/data.yaml'
+    model                = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ConvBasic_withEfficientKAN.py'
+    model_config         = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ConvBasic_withEfficientKAN.yaml'
+    preprocessing_config = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/ibis/ibis_SP140/preprocessing.yaml'
+    genome               = 'GRCh38'
+    bed_peak_size        = 40
+
+    // output
+    save_data    = true
+}
+
+// Limit resources so that this can run on GitHub Actions
+process {
+    maxRetries = params.max_retries
+    errorStrategy = params.err_start
+
+    withLabel:process_low {
+        cpus   = { 1                    }
+        memory = { 4.GB * task.attempt  }
+        time   = { 10.m  * task.attempt }
+    }
+    withLabel:process_medium {
+        cpus   = { 2                    }
+        memory = { 6.GB * task.attempt  }
+        time   = { 30.m  * task.attempt }
+    }
+    withLabel:process_high {
+        cpus   = { 4                    }
+        memory = { 8.GB * task.attempt  }
+        time   = { 1.h  * task.attempt  }
+    }
+}

conf/test_noise_eval.config

@@ -0,0 +1,54 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/rnaseq -profile test,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'Test profile'
+    config_profile_description = 'Minimal test dataset to check pipeline functions'
+
+    // Input data
+    data         = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_stimulus.csv'
+    data_config  = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic.yaml'
+    model        = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_model.py'
+    model_config = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_model.yaml'
+
+    // tune parameters
+    tune_trials_range = "2,4,2"
+    tune_replicates   = 2
+
+    // predict data
+    prediction_data = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_stimulus.csv'
+
+    // output
+    save_data    = false
+}
+
+// Limit resources so that this can run on GitHub Actions
+process {
+    maxRetries = params.max_retries
+    errorStrategy = params.err_start
+
+    withLabel:process_low {
+        cpus   = { 1                    }
+        memory = { 4.GB * task.attempt  }
+        time   = { 10.m  * task.attempt }
+    }
+    withLabel:process_medium {
+        cpus   = { 2                    }
+        memory = { 6.GB * task.attempt  }
+        time   = { 30.m  * task.attempt }
+    }
+    withLabel:process_high {
+        cpus   = { 4                    }
+        memory = { 8.GB * task.attempt  }
+        time   = { 1.h  * task.attempt  }
+    }
+}