Skip to content

Commit f520da2

Browse files
suzannejinsuzannejin
committed
merge and resolve conflict
2 parents 1e9d8e1 + f6033a0 commit f520da2

File tree

36 files changed

+1129
-201
lines changed

36 files changed

+1129
-201
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ jobs:
3737
- "test"
3838
- "test_ibis"
3939
- "test_ibis_with_preprocessing"
40+
- "test_noise_eval"
4041
isMaster:
4142
- ${{ github.base_ref == 'master' }}
4243
# Exclude conda and singularity on dev

‎conf/modules.config‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ process {
7575
publishDir = [
7676
path: {"${params.outdir}/data"},
7777
mode: params.publish_dir_mode,
78-
pattern: "*.csv"
78+
pattern: "*.csv",
79+
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
7980
]
8081
ext.prefix = { "${meta.id}.stimulus" }
8182
ext.suffix = { "csv" }
@@ -91,9 +92,11 @@ process {
9192

9293
withName: "STIMULUS_SPLIT_TRANSFORM" {
9394
publishDir = [
94-
path: { "${params.outdir}/configs" },
95-
mode: params.publish_dir_mode
95+
path: { "${params.outdir}/configs/${meta.id}" },
96+
mode: params.publish_dir_mode,
97+
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
9698
]
99+
tag = { "${meta.id}" }
97100
if (params.container_dev) {
98101
container = params.container_dev
99102
}
@@ -104,18 +107,21 @@ process {
104107
enabled: false
105108
]
106109
ext.prefix = { "${meta.id}-split-${meta.split_id}" }
110+
tag = { "${meta.id} - split: ${meta.split_id}" }
107111
if (params.container_dev) {
108112
container = params.container_dev
109113
}
110114
}
111115

112116
withName: "STIMULUS_TRANSFORM_CSV" {
113117
publishDir = [
114-
path: { "${params.outdir}/transformed_data" },
118+
path: { "${params.outdir}/transformed_data/${meta.id}" },
115119
mode: params.publish_dir_mode,
116-
when: params.save_data
120+
when: params.save_data,
121+
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
117122
]
118123
ext.prefix = { "${meta.id}-split-${meta.split_id}-trans-${meta.transform_id}" }
124+
tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id}" }
119125
if (params.container_dev) {
120126
container = params.container_dev
121127
}
@@ -126,6 +132,7 @@ process {
126132
params.check_model_num_samples ? "-n ${params.check_model_num_samples}" : '',
127133
params.debug_mode ? "--debug_mode" : ''
128134
].flatten().unique(false).join(' ').trim()}
135+
tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id}" }
129136
if (params.container_dev) {
130137
container = params.container_dev
131138
}
@@ -136,12 +143,39 @@ process {
136143
params.debug_mode ? "--debug_mode" : ''
137144
].flatten().unique(false).join(' ').trim()}
138145
publishDir = [
139-
path: { "${params.outdir}/tune_results/${meta.id}-split-${meta.split_id}-trans-${meta.transform_id}" },
140-
mode: params.publish_dir_mode
146+
path: { "${params.outdir}/tune_results/${meta.id}/${meta.split_id}/${meta.transform_id}" },
147+
mode: params.publish_dir_mode,
148+
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
141149
]
150+
ext.prefix = { "${meta.id}-split-${meta.split_id}-trans-${meta.transform_id}-rep-${meta.replicate}" }
151+
tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id} - replicate: ${meta.replicate}" }
142152
if (params.container_dev) {
143153
container = params.container_dev
144154
}
145155
}
156+
157+
withName: "STIMULUS_PREDICT" {
158+
ext.prefix = { "${meta.id}-split-${meta.split_id}-transf-${meta.transform_id}-rep-${meta.replicate}" }
159+
tag = { "${meta.id} - split: ${meta.split_id} - transform: ${meta.transform_id} - replicate: ${meta.replicate}" }
160+
if (params.container_dev) {
161+
container = params.container_dev
162+
}
163+
}
164+
165+
withName: "STIMULUS_COMPARE_TENSORS_COSINE" {
166+
ext.args = { "--mode cosine_similarity" }
167+
if (params.container_dev) {
168+
container = params.container_dev
169+
}
170+
}
171+
172+
withName: "CONCAT_COSINE" {
173+
ext.prefix = { "summary_cosine" }
174+
publishDir = [
175+
path: { "${params.outdir}/evaluation_results/" },
176+
mode: params.publish_dir_mode,
177+
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
178+
]
179+
}
146180
}
147181

‎conf/test_noise_eval.config‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/rnaseq -profile test,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
params {
14+
config_profile_name = 'Test profile'
15+
config_profile_description = 'Minimal test dataset to check pipeline functions'
16+
17+
// Input data
18+
data = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_stimulus.csv'
19+
data_config = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic.yaml'
20+
model = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_model.py'
21+
model_config = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_model.yaml'
22+
23+
// tune parameters
24+
tune_trials_range = "2,4,2"
25+
tune_replicates = 2
26+
27+
// predict data
28+
prediction_data = params.pipelines_testdata_base_path + 'deepmodeloptim/testdata/titanic/titanic_stimulus.csv'
29+
30+
// output
31+
save_data = false
32+
}
33+
34+
// Limit resources so that this can run on GitHub Actions
35+
process {
36+
maxRetries = params.max_retries
37+
errorStrategy = params.err_start
38+
39+
withLabel:process_low {
40+
cpus = { 1 }
41+
memory = { 4.GB * task.attempt }
42+
time = { 10.m * task.attempt }
43+
}
44+
withLabel:process_medium {
45+
cpus = { 2 }
46+
memory = { 6.GB * task.attempt }
47+
time = { 30.m * task.attempt }
48+
}
49+
withLabel:process_high {
50+
cpus = { 4 }
51+
memory = { 8.GB * task.attempt }
52+
time = { 1.h * task.attempt }
53+
}
54+
}

‎examples/pipeline_generated.json‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎examples/test.json‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎examples/user_given.json‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

‎main.nf‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ workflow NFCORE_DEEPMODELOPTIM {
3838
initial_weights
3939
preprocessing_config
4040
genome
41+
tune_trials_range
42+
tune_replicates
43+
prediction_data
4144

4245
main:
4346

@@ -51,7 +54,10 @@ workflow NFCORE_DEEPMODELOPTIM {
5154
model_config,
5255
initial_weights,
5356
preprocessing_config,
54-
genome
57+
genome,
58+
tune_trials_range,
59+
tune_replicates,
60+
prediction_data
5561
)
5662
}
5763

@@ -92,7 +98,10 @@ workflow {
9298
PIPELINE_INITIALISATION.out.model_config,
9399
PIPELINE_INITIALISATION.out.initial_weights,
94100
PIPELINE_INITIALISATION.out.preprocessing_config,
95-
PIPELINE_INITIALISATION.out.genome
101+
PIPELINE_INITIALISATION.out.genome,
102+
PIPELINE_INITIALISATION.out.tune_trials_range,
103+
PIPELINE_INITIALISATION.out.tune_replicates,
104+
PIPELINE_INITIALISATION.out.prediction_data
96105
)
97106

98107
//

‎modules.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@
2525
"git_sha": "81880787133db07d9b4c1febd152c090eb8325dc",
2626
"installed_by": ["modules"]
2727
},
28+
"csvtk/concat": {
29+
"branch": "master",
30+
"git_sha": "fa92936611247ca647ddee970f94d83f1bcb0ffe",
31+
"installed_by": ["modules"]
32+
},
2833
"custom/getchromsizes": {
2934
"branch": "master",
3035
"git_sha": "81880787133db07d9b4c1febd152c090eb8325dc",

‎modules/local/custom/modify_model_config/main.nf‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
process CUSTOM_MODIFY_MODEL_CONFIG {
2+
3+
tag "${meta.id} - #trial ${n_trials}"
4+
label 'process_low'
5+
6+
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
7+
'https://depot.galaxyproject.org/singularity/ubuntu:22.04' :
8+
'nf-core/ubuntu:22.04' }"
9+
10+
input:
11+
tuple val(meta), path(config)
12+
val(n_trials)
13+
14+
output:
15+
tuple val(meta_updated), path("${prefix}.yaml"), emit: config
16+
path "versions.yml" , emit: versions
17+
18+
script:
19+
prefix = task.ext.prefix ?: "${config.baseName}-trials_updated"
20+
meta_updated = meta + ["n_trials": "${n_trials}"]
21+
"""
22+
# substitte the line containing n_trials in the config file with n_trials: \${n_trials}
23+
awk -v n_trials=${n_trials} '/n_trials: [0-9]+/ {gsub(/n_trials: [0-9]+/, "n_trials: " n_trials)}1' ${config} > ${prefix}.yaml
24+
25+
cat <<-END_VERSIONS > versions.yml
26+
"${task.process}":
27+
bash: \$(echo \$(bash --version | grep -Eo 'version [[:alnum:].]+' | sed 's/version //'))
28+
END_VERSIONS
29+
"""
30+
31+
stub:
32+
prefix = task.ext.prefix ?: "${meta.id}"
33+
"""
34+
touch ${prefix}.yaml
35+
36+
cat <<-END_VERSIONS > versions.yml
37+
"${task.process}":
38+
bash: \$(echo \$(bash --version | grep -Eo 'version [[:alnum:].]+' | sed 's/version //'))
39+
END_VERSIONS
40+
41+
"""
42+
}

‎modules/local/stimulus/check_model/main.nf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
process CHECK_MODEL {
22

3-
tag "check model"
3+
tag "${meta.id}"
44
label 'process_medium'
55
// TODO: push image to nf-core quay.io
66
container "docker.io/mathysgrapotte/stimulus-py:dev"

0 commit comments

Comments
 (0)