From cec17192389c133235618c50a48d2e7afefbf520 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Thu, 30 Oct 2025 13:17:32 -0700 Subject: [PATCH 01/21] Added fq2bam --- .../nf-core/fastq_align_dedup_bwamem/main.nf | 57 +++- .../nf-core/fastq_align_dedup_bwamem/meta.yml | 8 +- .../fastq_align_dedup_bwamem/nextflow.config | 1 - .../tests/gpu.nf.test | 187 +++++++++++ .../tests/main.nf.test | 29 +- .../tests/main.nf.test.snap | 291 +++++------------- .../tests/nextflow.config | 3 + tests/config/nf-test.config | 2 +- 8 files changed, 330 insertions(+), 248 deletions(-) mode change 100644 => 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf mode change 100644 => 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml mode change 100644 => 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/nextflow.config create mode 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test mode change 100644 => 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test mode change 100644 => 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap mode change 100644 => 100755 subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf old mode 100644 new mode 100755 index a496e1e3b554..2deb22419b12 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf @@ -1,6 +1,8 @@ -include { FASTQ_ALIGN_BWA } from '../fastq_align_bwa/main' +include { BAM_SORT_STATS_SAMTOOLS } from '../../nf-core/bam_sort_stats_samtools/main' +include { FASTQ_ALIGN_BWA } from '../../nf-core/fastq_align_bwa/main' include { PICARD_ADDORREPLACEREADGROUPS } from '../../../modules/nf-core/picard/addorreplacereadgroups/main' include { PICARD_MARKDUPLICATES } from '../../../modules/nf-core/picard/markduplicates/main' +include { PARABRICKS_FQ2BAM } from '../../../modules/nf-core/parabricks/fq2bam/main' include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' workflow FASTQ_ALIGN_DEDUP_BWAMEM { @@ -23,18 +25,47 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { ch_multiqc_files = Channel.empty() ch_versions = Channel.empty() - FASTQ_ALIGN_BWA ( - ch_reads, - ch_bwamem_index, - true, // val_sort_bam hardcoded to true - ch_fasta - ) - ch_alignment = ch_alignment.mix(FASTQ_ALIGN_BWA.out.bam) - ch_alignment_index = ch_alignment.mix(FASTQ_ALIGN_BWA.out.bai) - ch_stats = ch_alignment.mix(FASTQ_ALIGN_BWA.out.stats) // channel: [ val(meta), path(stats) ] - ch_flagstat = ch_alignment.mix(FASTQ_ALIGN_BWA.out.flagstat) // channel: [ val(meta), path(flagstat) ] - ch_idxstats = ch_alignment.mix(FASTQ_ALIGN_BWA.out.idxstats) // channel: [ val(meta), path(idxstats) ] - ch_versions = ch_versions.mix(FASTQ_ALIGN_BWA.out.versions.first()) + if (use_gpu) { + /* + * Align with parabricks GPU enabled fq2bammeth implementation of bwameth + */ + PARABRICKS_FQ2BAM ( + ch_reads, + ch_fasta, + ch_bwamem_index, + [[],[]], // interval file + [[],[]], // known sites + 'bam' // output format + ) + ch_alignment = PARABRICKS_FQ2BAM.out.bam + ch_versions = ch_versions.mix(PARABRICKS_FQ2BAM.out.versions.first()) + + // FQ2BAM can also sort and markduplicates + BAM_SORT_STATS_SAMTOOLS ( + ch_alignment, + ch_fasta + ) + ch_alignment = BAM_SORT_STATS_SAMTOOLS.out.bam + ch_alignment_index = BAM_SORT_STATS_SAMTOOLS.out.bai + ch_stats = BAM_SORT_STATS_SAMTOOLS.out.stats // channel: [ val(meta), path(stats) ] + ch_flagstat = BAM_SORT_STATS_SAMTOOLS.out.flagstat // channel: [ val(meta), path(flagstat) ] + ch_idxstats = BAM_SORT_STATS_SAMTOOLS.out.idxstats // channel: [ val(meta), path(idxstats) ] + ch_versions = ch_versions.mix(BAM_SORT_STATS_SAMTOOLS.out.versions.first()) + } + else { + FASTQ_ALIGN_BWA ( + ch_reads, + ch_bwamem_index, + true, // val_sort_bam hardcoded to true + ch_fasta + ) + ch_alignment = ch_alignment.mix(FASTQ_ALIGN_BWA.out.bam) + ch_alignment_index = ch_alignment.mix(FASTQ_ALIGN_BWA.out.bai) + ch_stats = ch_alignment.mix(FASTQ_ALIGN_BWA.out.stats) // channel: [ val(meta), path(stats) ] + ch_flagstat = ch_alignment.mix(FASTQ_ALIGN_BWA.out.flagstat) // channel: [ val(meta), path(flagstat) ] + ch_idxstats = ch_alignment.mix(FASTQ_ALIGN_BWA.out.idxstats) // channel: [ val(meta), path(idxstats) ] + ch_versions = ch_versions.mix(FASTQ_ALIGN_BWA.out.versions.first()) + } if (!skip_deduplication) { /* diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml b/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml old mode 100644 new mode 100755 index 359f2db66b01..c898194d6fb1 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json name: "fastq_align_dedup_bwamem" -description: Performs alignment of DNA or TAPS-treated reads using bwamem, sort and deduplicate +description: Performs alignment of DNA or TAPS-treated reads using bwamem or parabricks/fq2bam, sort and deduplicate keywords: - bwamem - alignment @@ -11,9 +11,11 @@ keywords: - fastq - bam components: + - parabricks/fq2bam - samtools/index - picard/addorreplacereadgroups - picard/markduplicates + - bam_sort_stats_samtools - fastq_align_bwa input: - ch_reads: @@ -41,6 +43,10 @@ input: type: boolean description: | Skip deduplication of aligned reads + - use_gpu: + type: boolean + description: | + Use GPU for alignment output: - bam: type: file diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bwamem/nextflow.config old mode 100644 new mode 100755 index bc1fcf151503..441288760beb --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/nextflow.config +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/nextflow.config @@ -1,4 +1,3 @@ -// IMPORTANT: This config file should be included to ensure that the subworkflow works properly. process { withName: 'SAMTOOLS_SORT' { ext.prefix = { "${meta.id}.sorted" } diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test new file mode 100755 index 000000000000..f5448c0ef719 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -0,0 +1,187 @@ +nextflow_workflow { + + name "Test Subworkflow FASTQ_ALIGN_DEDUP_BWAMEM" + script "../main.nf" + workflow "FASTQ_ALIGN_DEDUP_BWAMEM" + config "./nextflow.config" + + tag "gpu" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fastq_align_dedup_bwamem" + tag "parabricks/fq2bam" + tag "samtools/index" + tag "picard/markduplicates" + tag "untar" + + setup { + run("BWA_INDEX") { + script "../../../../modules/nf-core/bwa/index/main.nf" + process { + """ + input[0] = Channel.of([ + [ id:'test' ], // meta map + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + """ + } + } + + run("BWA_INDEX", alias: 'BWA_INDEX_PE') { + script "../../../../modules/nf-core/bwa/index/main.nf" + process { + """ + input[0] = Channel.of([ + [ id:'test' ], // meta map + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + """ + } + } + } + + test("Params: parabricks/fq2bam single-end | use_gpu ") { + when { + params { + use_gpu = true + bwa_prefix = 'genome.fa' + } + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) + ]) + input[3] = BWA_INDEX.out.index + input[4] = false // skip_deduplication + input[5] = true // use_gpu + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } + + test("Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication") { + when { + params { + skip_deduplication = true + use_gpu = true + bwa_prefix = 'genome.fa' + } + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) + ]) + input[3] = BWA_INDEX.out.index + input[4] = true // skip_deduplication + input[5] = true // use_gpu + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out, + workflow.out.versions.collect{ path(it).yaml }.unique() + ).match() } + ) + } + } + + test("Params: parabricks/fq2bam single-end | use_gpu | stub") { + options '-stub' + when { + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) + ]) + input[3] = BWA_INDEX.out.index + input[4] = false // skip_deduplication + input[5] = true // use_gpu + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } + + test("Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication") { + + when { + params { + skip_deduplication = true + use_gpu = true + bwa_prefix = 'genome.fa' + }sa + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) + ]) + input[3] = BWA_INDEX_PE.out.index + input[4] = true // skip_deduplication + input[5] = true // use_gpu + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test old mode 100644 new mode 100755 index be79feed6178..2e9315e919a2 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test @@ -8,19 +8,17 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" tag "subworkflows/fastq_align_dedup_bwamem" - tag "subworkflows/fastq_align_bwa" tag "bwa/index" - tag "bwa/mem" - tag "samtools" + tag "parabricks/fq2bam" tag "samtools/sort" tag "samtools/index" tag "samtools/flagstat" tag "samtools/stats" - tag "samtools/idxstats" tag "bam_sort_stats_samtools" tag "fastq_align_bwa" tag "picard/markduplicates" tag "picard/addorreplacereadgroups" + tag "samtools/index" tag "untar" setup { @@ -28,10 +26,9 @@ nextflow_workflow { script "../../../../modules/nf-core/bwa/index/main.nf" process { """ - input[0] = [ - [ id:'genome' ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] + input[0] = Channel.value([ [ id:'genome' ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } @@ -107,6 +104,7 @@ nextflow_workflow { then { assertAll( + { assert workflow.success }, { assert workflow.success }, { assert snapshot( workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, @@ -150,6 +148,7 @@ nextflow_workflow { then { assertAll( + { assert workflow.success }, { assert workflow.success }, { assert snapshot( workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, @@ -193,10 +192,16 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot( - workflow.out, - workflow.out.versions.collect{ path(it).yaml } - ).match() } + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.samtools_index_stats, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } ) } } diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap old mode 100644 new mode 100755 index 214e37f4d4d3..ab681fdbf12f --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap @@ -78,7 +78,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-24T15:06:49.6951852" + "timestamp": "2025-10-21T15:21:11.425903227" }, "Params: bwamem paired-end - skip_deduplication": { "content": [ @@ -156,237 +156,88 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-24T15:07:14.224268922" + "timestamp": "2025-10-21T15:21:38.976160156" }, "Params: bwamem single-end - default - stub": { "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "1": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "2": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ], - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "3": [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "4": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "5": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "6": [ - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ], - [ - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "7": [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,8f2ff4c430c4c5f42631534eaf358833", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" - ], - "bai": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "bam": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "multiqc": [ - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ], - [ - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "picard_metrics": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "samtools_flagstat": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ], - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + [ + "d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.deduped.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" ], - "samtools_index_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ], - "samtools_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" ], - "versions": [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,8f2ff4c430c4c5f42631534eaf358833", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] - }, - [ - { - "FASTQ_ALIGN_DEDUP_BWAMEM:PICARD_MARKDUPLICATES": { - "picard": "3.4.0" - } - }, - { - "FASTQ_ALIGN_DEDUP_BWAMEM:FASTQ_ALIGN_BWA:BWA_MEM": { - "bwa": "0.7.19-r1273", - "samtools": "1.22.1" - } - }, - { - "FASTQ_ALIGN_DEDUP_BWAMEM:SAMTOOLS_INDEX": { - "samtools": "1.22.1" - } - }, - { - "FASTQ_ALIGN_DEDUP_BWAMEM:PICARD_ADDORREPLACEREADGROUPS": { - "picard": "3.4.0" - } - } + ], + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt" + ], + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt", + "test.sorted.bam", + "test.flagstat", + "test.sorted.bam", + "test.idxstats", + "test.sorted.bam", + "test.stats" + ], + [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,8f2ff4c430c4c5f42631534eaf358833", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-24T15:07:27.497220186" + "timestamp": "2025-10-21T15:21:53.551546233" }, "Params: bwamem paired-end - default": { "content": [ @@ -467,6 +318,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-24T15:07:05.175157122" + "timestamp": "2025-10-21T15:21:28.657077612" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config old mode 100644 new mode 100755 index 755ba1b3feed..bd5ccae7b402 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config @@ -1,4 +1,7 @@ process { + withName: 'PARABRICKS_FQ2BAM' { + ext.args = '--low-memory' + } withName: 'SAMTOOLS_SORT' { ext.prefix = { "${meta.id}.sorted" } } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 336f2e178917..a8269b4a05f4 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -80,7 +80,7 @@ profiles { executor.memory = 8.GB } gpu { - docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + docker.runOptions = '-u $(id -u):$(id -g) --gpus all --memory=64g --memory-swap=64g' apptainer.runOptions = '--nv' singularity.runOptions = '--nv' } From e2a4e73ef57b868595e1c1f13d8c0ab155377988 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Thu, 30 Oct 2025 13:25:24 -0700 Subject: [PATCH 02/21] test fix --- subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index f5448c0ef719..db3f34679cb1 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -158,7 +158,7 @@ nextflow_workflow { workflow { """ input[0] = Channel.of([ - [ id:'test', single_end:true ], + [ id:'test', single_end:false ], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ]) From 548bcaef06879db57f4f4db73a161eaa6e50795a Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Thu, 30 Oct 2025 13:30:26 -0700 Subject: [PATCH 03/21] typo --- subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index db3f34679cb1..c2aaa09b0f74 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -154,7 +154,7 @@ nextflow_workflow { skip_deduplication = true use_gpu = true bwa_prefix = 'genome.fa' - }sa + } workflow { """ input[0] = Channel.of([ From 222cdc65d9b7ca9f0a017f4d2f2a96b55b85bf58 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Thu, 30 Oct 2025 13:36:04 -0700 Subject: [PATCH 04/21] PE channel --- .../nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index c2aaa09b0f74..3fdf34bd6819 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -158,9 +158,11 @@ nextflow_workflow { workflow { """ input[0] = Channel.of([ - [ id:'test', single_end:false ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + [ id:'test', single_end:false ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] ]) input[1] = Channel.of([ [:], From f3b9ad7934455b2bd167c9ced3bcd172d4a6bbc6 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Thu, 30 Oct 2025 13:44:21 -0700 Subject: [PATCH 05/21] tidying --- subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf | 1 - .../nf-core/fastq_align_dedup_bwamem/tests/main.nf.test | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf index 2deb22419b12..b80d9535195b 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf @@ -40,7 +40,6 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { ch_alignment = PARABRICKS_FQ2BAM.out.bam ch_versions = ch_versions.mix(PARABRICKS_FQ2BAM.out.versions.first()) - // FQ2BAM can also sort and markduplicates BAM_SORT_STATS_SAMTOOLS ( ch_alignment, ch_fasta diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test index 2e9315e919a2..b573ed4a8541 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test @@ -18,7 +18,6 @@ nextflow_workflow { tag "fastq_align_bwa" tag "picard/markduplicates" tag "picard/addorreplacereadgroups" - tag "samtools/index" tag "untar" setup { @@ -27,7 +26,7 @@ nextflow_workflow { process { """ input[0] = Channel.value([ [ id:'genome' ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) """ } From fc3bd64bd75b42f726c64ab0a0a10af7ddb52858 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 30 Oct 2025 20:47:47 +0000 Subject: [PATCH 06/21] gpu snap --- .../tests/gpu.nf.test.snap | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap new file mode 100644 index 000000000000..d01d58657957 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap @@ -0,0 +1,610 @@ +{ + "Params: parabricks/fq2bam single-end | use_gpu | stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "5": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "6": [ + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ], + [ + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "7": [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ], + "bai": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "multiqc": [ + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ], + [ + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "picard_metrics": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "samtools_flagstat": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ] + ], + "samtools_index_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "samtools_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.04.7" + }, + "timestamp": "2025-10-30T20:46:38.794044618" + }, + "Params: parabricks/fq2bam single-end | use_gpu ": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam:md5,90bbec5992bdef9485112411d848e490" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ] + ], + "5": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" + ] + ], + "6": [ + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" + ], + [ + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ], + [ + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ], + [ + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "7": [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ], + "bai": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam:md5,90bbec5992bdef9485112411d848e490" + ] + ], + "multiqc": [ + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" + ], + [ + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ], + [ + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ], + [ + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "picard_metrics": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" + ] + ], + "samtools_flagstat": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ] + ], + "samtools_index_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ] + ], + "samtools_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "versions": [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.04.7" + }, + "timestamp": "2025-10-30T20:45:35.858091961" + }, + "Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam:md5,dbba71a9cbc20a3955c511281412800b" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.bai:md5,16615e11435236323497a687daee78bc" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": false + }, + "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": false + }, + "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": false + }, + "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" + ] + ], + "5": [ + + ], + "6": [ + [ + "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" + ], + [ + "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" + ], + [ + "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" + ] + ], + "7": [ + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ], + "bai": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.bai:md5,16615e11435236323497a687daee78bc" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam:md5,dbba71a9cbc20a3955c511281412800b" + ] + ], + "multiqc": [ + [ + "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" + ], + [ + "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" + ], + [ + "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" + ] + ], + "picard_metrics": [ + + ], + "samtools_flagstat": [ + [ + { + "id": "test", + "single_end": false + }, + "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" + ] + ], + "samtools_index_stats": [ + [ + { + "id": "test", + "single_end": false + }, + "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" + ] + ], + "samtools_stats": [ + [ + { + "id": "test", + "single_end": false + }, + "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" + ] + ], + "versions": [ + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.04.7" + }, + "timestamp": "2025-10-30T20:47:26.4261594" + }, + "Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,58a20b8286a1b8582ee08966db328960" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" + ] + ], + "2": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ] + ], + "5": [ + + ], + "6": [ + [ + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ], + [ + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ], + [ + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "7": [ + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ], + "bai": [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,58a20b8286a1b8582ee08966db328960" + ] + ], + "multiqc": [ + [ + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ], + [ + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ], + [ + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "picard_metrics": [ + + ], + "samtools_flagstat": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" + ] + ], + "samtools_index_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ] + ], + "samtools_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + "versions": [ + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ] + }, + [ + { + "FASTQ_ALIGN_DEDUP_BWAMEM:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT": { + "samtools": "1.22.1" + } + }, + { + "FASTQ_ALIGN_DEDUP_BWAMEM:PARABRICKS_FQ2BAM": { + "pbrun": "4.6.0-1" + } + } + ] + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.04.7" + }, + "timestamp": "2025-10-30T20:46:23.389686638" + } +} \ No newline at end of file From 7688d5b53cd3262d8268d12008ab738c4085bf6b Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Thu, 30 Oct 2025 14:05:56 -0700 Subject: [PATCH 07/21] test fix for snaps --- .../tests/gpu.nf.test | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index 3fdf34bd6819..a6f397d99d26 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -70,7 +70,17 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.samtools_index_stats, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() + } ) } } @@ -107,9 +117,16 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out, - workflow.out.versions.collect{ path(it).yaml }.unique() - ).match() } + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.samtools_index_stats, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() + } ) } } @@ -182,7 +199,17 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.samtools_index_stats, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() + } ) } } From 3a4c54bb67b81394b033041331a866ee2f6911a1 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 30 Oct 2025 21:12:19 +0000 Subject: [PATCH 08/21] gpu snapshot --- .../tests/gpu.nf.test.snap | 557 +++++------------- 1 file changed, 138 insertions(+), 419 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap index d01d58657957..234d0ba12f07 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap @@ -158,453 +158,172 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T20:46:38.794044618" + "timestamp": "2025-10-30T21:08:22.67936189" }, "Params: parabricks/fq2bam single-end | use_gpu ": { "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam:md5,90bbec5992bdef9485112411d848e490" - ] - ], - "1": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" - ] - ], - "2": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ] - ], - "3": [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "4": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ] - ], - "5": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" - ] - ], - "6": [ - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" - ], - [ - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ], - [ - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ], - [ - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "7": [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" - ], - "bai": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" - ] - ], - "bam": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam:md5,90bbec5992bdef9485112411d848e490" - ] - ], - "multiqc": [ - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" - ], - [ - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ], - [ - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ], - [ - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "picard_metrics": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,9939e25cd7085969f3ce045e3a144e09" - ] - ], - "samtools_flagstat": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ] - ], - "samtools_index_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ] - ], - "samtools_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "versions": [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + [ + "da777b5d079e09dfc076b1c89f93ef3e" + ], + [ + "test.deduped.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" ] - } + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ] + ], + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt" + ], + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt", + "test.flagstat", + "test.idxstats", + "test.stats" + ], + [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ] ], "meta": { "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T20:45:35.858091961" + "timestamp": "2025-10-30T21:07:20.05203529" }, "Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication": { "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.sorted.bam:md5,dbba71a9cbc20a3955c511281412800b" - ] - ], - "1": [ - [ - { - "id": "test", - "single_end": false - }, - "test.sorted.bam.bai:md5,16615e11435236323497a687daee78bc" - ] - ], - "2": [ - [ - { - "id": "test", - "single_end": false - }, - "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" - ] - ], - "3": [ - [ - { - "id": "test", - "single_end": false - }, - "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" - ] - ], - "4": [ - [ - { - "id": "test", - "single_end": false - }, - "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" - ] - ], - "5": [ - - ], - "6": [ - [ - "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" - ], - [ - "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" - ], - [ - "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" - ] - ], - "7": [ - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" - ], - "bai": [ - [ - { - "id": "test", - "single_end": false - }, - "test.sorted.bam.bai:md5,16615e11435236323497a687daee78bc" - ] - ], - "bam": [ - [ - { - "id": "test", - "single_end": false - }, - "test.sorted.bam:md5,dbba71a9cbc20a3955c511281412800b" - ] - ], - "multiqc": [ - [ - "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" - ], - [ - "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" - ], - [ - "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" - ] - ], - "picard_metrics": [ - - ], - "samtools_flagstat": [ - [ - { - "id": "test", - "single_end": false - }, - "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" - ] - ], - "samtools_index_stats": [ - [ - { - "id": "test", - "single_end": false - }, - "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" - ] - ], - "samtools_stats": [ - [ - { - "id": "test", - "single_end": false - }, - "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" - ] - ], - "versions": [ - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + [ + "50a751c2aaaaeaa8c341d948aaa9cc0d" + ], + [ + "test.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" ] - } + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" + ] + ], + [ + + ], + [ + "test.flagstat", + "test.idxstats", + "test.stats" + ], + [ + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ] ], "meta": { "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T20:47:26.4261594" + "timestamp": "2025-10-30T21:09:10.053957182" }, "Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication": { "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,58a20b8286a1b8582ee08966db328960" - ] - ], - "1": [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" - ] - ], - "2": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ] - ], - "3": [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "4": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ] - ], - "5": [ - - ], - "6": [ - [ - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ], - [ - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ], - [ - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "7": [ - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" - ], - "bai": [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam.bai:md5,3b99d4c5af1dfc3b19d938ac5e9a5b77" - ] - ], - "bam": [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,58a20b8286a1b8582ee08966db328960" - ] - ], - "multiqc": [ - [ - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ], - [ - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ], - [ - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "picard_metrics": [ - - ], - "samtools_flagstat": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ] - ], - "samtools_index_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ] - ], - "samtools_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] - ], - "versions": [ - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + [ + "da777b5d079e09dfc076b1c89f93ef3e" + ], + [ + "test.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" ] - }, + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,ee6142f20133332ffa6d94726116be3b" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" + ] + ], + [ + + ], + [ + "test.flagstat", + "test.idxstats", + "test.stats" + ], [ - { - "FASTQ_ALIGN_DEDUP_BWAMEM:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT": { - "samtools": "1.22.1" - } - }, - { - "FASTQ_ALIGN_DEDUP_BWAMEM:PARABRICKS_FQ2BAM": { - "pbrun": "4.6.0-1" - } - } + "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", + "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" ] ], "meta": { "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T20:46:23.389686638" + "timestamp": "2025-10-30T21:08:07.273242225" } } \ No newline at end of file From 0201ccbe48b7abb1cd47a7d7bb103fd42d3db4e5 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Fri, 31 Oct 2025 09:22:52 -0600 Subject: [PATCH 09/21] PR feedback --- subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf | 2 ++ subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml | 10 ++++++++++ .../fastq_align_dedup_bwameth/tests/nextflow.config | 3 ++- tests/config/nf-test.config | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf index b80d9535195b..b4cc14a2f90c 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf @@ -14,6 +14,8 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { ch_bwamem_index // channel: [ val(meta), [ bwam index ] ] skip_deduplication // boolean: whether to deduplicate alignments use_gpu // boolean: whether to use GPU or CPU for bwamem alignment + interval_file // channel: [ val(meta), [ interval file ] ] + known_sites // channel: [ val(meta), [ known sites ] ] main: diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml b/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml index c898194d6fb1..cff5cf736ec9 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/meta.yml @@ -47,6 +47,16 @@ input: type: boolean description: | Use GPU for alignment + - interval_file: + type: file + description: | + Structure: [ val(meta), path(interval file) ] + pattern: "*.{bed,intervals}" + - known_sites: + type: file + description: | + Structure: [ val(meta), path(known sites) ] + pattern: "*.{vcf,vcf.gz}" output: - bam: type: file diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config index 55385ec0621e..fbbf1509df20 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config @@ -1,6 +1,7 @@ process { withName: 'PARABRICKS_FQ2BAMMETH' { ext.args = '--low-memory' + containerOptions = '--gpus all --memory=64g --memory-swap=64g' } withName: 'SAMTOOLS_SORT' { @@ -11,4 +12,4 @@ process { ext.args = "--ASSUME_SORTED true --REMOVE_DUPLICATES false --VALIDATION_STRINGENCY LENIENT --PROGRAM_RECORD_ID 'null' --TMP_DIR tmp" ext.prefix = { "${meta.id}.markdup.sorted" } } -} +} \ No newline at end of file diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index a8269b4a05f4..336f2e178917 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -80,7 +80,7 @@ profiles { executor.memory = 8.GB } gpu { - docker.runOptions = '-u $(id -u):$(id -g) --gpus all --memory=64g --memory-swap=64g' + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' apptainer.runOptions = '--nv' singularity.runOptions = '--nv' } From 61a9670cd1a56b87f49478b07af0544a4d943de7 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Fri, 31 Oct 2025 09:27:21 -0600 Subject: [PATCH 10/21] tests --- .../nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test | 8 ++++++++ .../nf-core/fastq_align_dedup_bwamem/tests/main.nf.test | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index a6f397d99d26..218534706d6f 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -63,6 +63,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // skip_deduplication input[5] = true // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } @@ -109,6 +111,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = true // skip_deduplication input[5] = true // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } @@ -152,6 +156,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // skip_deduplication input[5] = true // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } @@ -192,6 +198,8 @@ nextflow_workflow { input[3] = BWA_INDEX_PE.out.index input[4] = true // skip_deduplication input[5] = true // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test index b573ed4a8541..5b9ee22da81f 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test @@ -54,6 +54,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // skip_deduplication input[5] = false // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } @@ -97,6 +99,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // skip_deduplication input[5] = false // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } @@ -141,6 +145,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = true // skip_deduplication input[5] = false // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } @@ -184,6 +190,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // skip_deduplication input[5] = false // use_gpu + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites """ } } From 4a214d0e7a705066c8218fa37d9180c8799551c3 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 31 Oct 2025 15:32:15 +0000 Subject: [PATCH 11/21] cpu snapshot --- .../fastq_align_dedup_bwamem/tests/main.nf.test.snap | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) mode change 100755 => 100644 subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap old mode 100755 new mode 100644 index ab681fdbf12f..ae1573a72264 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap @@ -78,7 +78,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-21T15:21:11.425903227" + "timestamp": "2025-10-31T15:27:53.091677319" }, "Params: bwamem paired-end - skip_deduplication": { "content": [ @@ -156,7 +156,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-21T15:21:38.976160156" + "timestamp": "2025-10-31T15:28:18.193321062" }, "Params: bwamem single-end - default - stub": { "content": [ @@ -237,7 +237,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-21T15:21:53.551546233" + "timestamp": "2025-10-31T15:28:31.607014969" }, "Params: bwamem paired-end - default": { "content": [ @@ -318,6 +318,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-21T15:21:28.657077612" + "timestamp": "2025-10-31T15:28:08.721629597" } } \ No newline at end of file From d30ae9a76bda6ec7cd422fe47a937edc365410cd Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Fri, 31 Oct 2025 09:38:21 -0600 Subject: [PATCH 12/21] tests --- .../nf-core/fastq_align_dedup_bwamem/tests/nextflow.config | 1 + .../nf-core/fastq_align_dedup_bwameth/tests/nextflow.config | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config index bd5ccae7b402..bb05b1ae6943 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/nextflow.config @@ -1,6 +1,7 @@ process { withName: 'PARABRICKS_FQ2BAM' { ext.args = '--low-memory' + containerOptions = '--gpus all --memory=64g --memory-swap=64g' } withName: 'SAMTOOLS_SORT' { ext.prefix = { "${meta.id}.sorted" } diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config index fbbf1509df20..621348c4b8ae 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config @@ -1,7 +1,6 @@ process { withName: 'PARABRICKS_FQ2BAMMETH' { ext.args = '--low-memory' - containerOptions = '--gpus all --memory=64g --memory-swap=64g' } withName: 'SAMTOOLS_SORT' { From 33a8c26b9acdfea9b3b4eef57617e584291db25d Mon Sep 17 00:00:00 2001 From: root Date: Fri, 31 Oct 2025 15:51:09 +0000 Subject: [PATCH 13/21] last snapshot --- .../fastq_align_dedup_bwamem/tests/gpu.nf.test.snap | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap index 234d0ba12f07..7a32bf73479e 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap @@ -158,7 +158,7 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T21:08:22.67936189" + "timestamp": "2025-10-31T15:50:07.241851782" }, "Params: parabricks/fq2bam single-end | use_gpu ": { "content": [ @@ -216,7 +216,7 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T21:07:20.05203529" + "timestamp": "2025-10-31T15:49:03.837208054" }, "Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication": { "content": [ @@ -270,7 +270,7 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T21:09:10.053957182" + "timestamp": "2025-10-31T15:50:55.102826956" }, "Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication": { "content": [ @@ -324,6 +324,6 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-30T21:08:07.273242225" + "timestamp": "2025-10-31T15:49:51.433758416" } } \ No newline at end of file From 509acdc016c1f31665e37da7c188bfc893a58195 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Fri, 31 Oct 2025 09:55:47 -0600 Subject: [PATCH 14/21] fix channels --- subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf index b4cc14a2f90c..7c29b43e84d3 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf @@ -35,8 +35,8 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { ch_reads, ch_fasta, ch_bwamem_index, - [[],[]], // interval file - [[],[]], // known sites + interval_file, // interval file + known_sites, // known sites 'bam' // output format ) ch_alignment = PARABRICKS_FQ2BAM.out.bam From e90793af584320ac2d36e13a656bc9f551aba0a4 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 31 Oct 2025 15:58:44 +0000 Subject: [PATCH 15/21] last snapshot --- .../tests/gpu.nf.test.snap | 253 +++--------------- 1 file changed, 42 insertions(+), 211 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap index 7a32bf73479e..8c85b6b0f6a7 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap @@ -3,154 +3,52 @@ "content": [ { "0": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "1": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "2": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ] + ], "3": [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "4": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "5": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "6": [ - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ], - [ - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "7": [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ], "bai": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "bam": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "multiqc": [ - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ], - [ - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "picard_metrics": [ - [ - { - "id": "test", - "single_end": true - }, - "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "samtools_flagstat": [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" - ] + ], "samtools_index_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "samtools_stats": [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "versions": [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ] } ], @@ -158,172 +56,105 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:50:07.241851782" + "timestamp": "2025-10-31T15:58:31.925828509" }, "Params: parabricks/fq2bam single-end | use_gpu ": { "content": [ [ - "da777b5d079e09dfc076b1c89f93ef3e" + ], [ - "test.deduped.sorted.bam.bai" + ], [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ] + ], [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] + ], [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ] + ], [ - "test.deduped.sorted.MarkDuplicates.metrics.txt" + ], [ - "test.deduped.sorted.MarkDuplicates.metrics.txt", - "test.flagstat", - "test.idxstats", - "test.stats" + ], [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ] ], "meta": { "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:49:03.837208054" + "timestamp": "2025-10-31T15:58:17.076913842" }, "Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication": { "content": [ [ - "50a751c2aaaaeaa8c341d948aaa9cc0d" + ], [ - "test.sorted.bam.bai" + ], [ - [ - { - "id": "test", - "single_end": false - }, - "test.flagstat:md5,c86d452cccb6cf4c98f18b024ffd208f" - ] + ], [ - [ - { - "id": "test", - "single_end": false - }, - "test.stats:md5,af3201af4fafc49bb1b5144675f85b5b" - ] + ], [ - [ - { - "id": "test", - "single_end": false - }, - "test.idxstats:md5,60ccb84f40d7c5e7e3c1775d54fe7f93" - ] + ], [ ], [ - "test.flagstat", - "test.idxstats", - "test.stats" + ], [ - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ] ], "meta": { "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:50:55.102826956" + "timestamp": "2025-10-31T15:58:39.347884795" }, "Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication": { "content": [ [ - "da777b5d079e09dfc076b1c89f93ef3e" + ], [ - "test.sorted.bam.bai" + ], [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,ba966c6410eccf2375c3bcc081ef4787" - ] + ], [ - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,ee6142f20133332ffa6d94726116be3b" - ] + ], [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,36588ec1a479d3e528b230ca70d0279d" - ] + ], [ ], [ - "test.flagstat", - "test.idxstats", - "test.stats" + ], [ - "versions.yml:md5,70c24af396804a56c49f53abe6da2d02", - "versions.yml:md5,71928d7ca7a14123b4b025bab027d798" + ] ], "meta": { "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:49:51.433758416" + "timestamp": "2025-10-31T15:58:24.519410831" } } \ No newline at end of file From d40d5b5391b3e3050768b2501993d61eed2035d4 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Mon, 3 Nov 2025 11:02:26 -0700 Subject: [PATCH 16/21] PR feedback --- .../nf-core/fastq_align_dedup_bwamem/main.nf | 3 +- .../tests/gpu.nf.test | 126 ++++++++---------- .../tests/main.nf.test | 21 ++- 3 files changed, 67 insertions(+), 83 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf index 7c29b43e84d3..6dae11abacda 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf @@ -14,6 +14,7 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { ch_bwamem_index // channel: [ val(meta), [ bwam index ] ] skip_deduplication // boolean: whether to deduplicate alignments use_gpu // boolean: whether to use GPU or CPU for bwamem alignment + output_fmt // string: output format for parabricks fq2bam (e.g., 'bam' or 'cram') interval_file // channel: [ val(meta), [ interval file ] ] known_sites // channel: [ val(meta), [ known sites ] ] @@ -37,7 +38,7 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { ch_bwamem_index, interval_file, // interval file known_sites, // known sites - 'bam' // output format + output_fmt // output format ) ch_alignment = PARABRICKS_FQ2BAM.out.bam ch_versions = ch_versions.mix(PARABRICKS_FQ2BAM.out.versions.first()) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index 218534706d6f..e897492d947d 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -12,7 +12,18 @@ nextflow_workflow { tag "parabricks/fq2bam" tag "samtools/index" tag "picard/markduplicates" - tag "untar" + tag "bwa" + tag "bwa/index" + tag "parabricks/fq2bam" + tag "samtools" + tag "samtools/sort" + tag "samtools/index" + tag "samtools/idxstats" + tag "samtools/flagstat" + tag "samtools/stats" + tag "bam_sort_stats_samtools" + tag "picard/markduplicates" + tag "picard/addorreplacereadgroups" setup { run("BWA_INDEX") { @@ -21,31 +32,15 @@ nextflow_workflow { """ input[0] = Channel.of([ [ id:'test' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) - ]) - """ - } - } - - run("BWA_INDEX", alias: 'BWA_INDEX_PE') { - script "../../../../modules/nf-core/bwa/index/main.nf" - process { - """ - input[0] = Channel.of([ - [ id:'test' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) """ } } } - test("Params: parabricks/fq2bam single-end | use_gpu ") { + test("Sarscov2 fasta - SE - deduplicate - with GPU parabricks/fq2bam") { when { - params { - use_gpu = true - bwa_prefix = 'genome.fa' - } workflow { """ input[0] = Channel.of([ @@ -61,10 +56,10 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = false // skip_deduplication - input[5] = true // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = false // deduplicate + input[5] = true // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites """ } } @@ -87,13 +82,8 @@ nextflow_workflow { } } - test("Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication") { + test("Sarscov2 fasta - SE - skip deduplication - with GPU parabricks/fq2bam") { when { - params { - skip_deduplication = true - use_gpu = true - bwa_prefix = 'genome.fa' - } workflow { """ input[0] = Channel.of([ @@ -109,10 +99,10 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = true // skip_deduplication - input[5] = true // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = true // skip_deduplication + input[5] = true // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites """ } } @@ -135,15 +125,17 @@ nextflow_workflow { } } - test("Params: parabricks/fq2bam single-end | use_gpu | stub") { - options '-stub' - when { + test("Sarscov2 fasta - PE - skip deduplication - with GPU parabricks/fq2bam") { + when { workflow { """ input[0] = Channel.of([ - [ id:'test', single_end:true ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + [ id:'test', single_end:false ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] ]) input[1] = Channel.of([ [:], @@ -154,10 +146,10 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = false // skip_deduplication - input[5] = true // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = true // skip_deduplication + input[5] = true // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites """ } } @@ -165,27 +157,29 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.samtools_index_stats, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() + } ) } } - test("Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication") { - + test("Sarscov2 fasta - SE - skip deduplication - with GPU parabricks/fq2bam - stub") { + options '-stub' when { - params { - skip_deduplication = true - use_gpu = true - bwa_prefix = 'genome.fa' - } workflow { """ input[0] = Channel.of([ - [ id:'test', single_end:false ], - [ - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) - ] + [ id:'test', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ]) input[1] = Channel.of([ [:], @@ -195,11 +189,11 @@ nextflow_workflow { [:], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) - input[3] = BWA_INDEX_PE.out.index - input[4] = true // skip_deduplication - input[5] = true // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[3] = BWA_INDEX.out.index + input[4] = false // deduplicate + input[5] = true // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites """ } } @@ -208,15 +202,9 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, - workflow.out.bai.collect { meta, bai -> file(bai).name }, - workflow.out.samtools_flagstat, - workflow.out.samtools_stats, - workflow.out.samtools_index_stats, - workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, - workflow.out.multiqc.flatten().collect { path -> file(path).name }, - workflow.out.versions - ).match() + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } ) } diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test index 5b9ee22da81f..eb4ea0caac28 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test @@ -8,17 +8,21 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" tag "subworkflows/fastq_align_dedup_bwamem" + tag "subworkflows/fastq_align_bwa" + tag "bwa" tag "bwa/index" + tag "bwa/mem" tag "parabricks/fq2bam" + tag "samtools" tag "samtools/sort" tag "samtools/index" + tag "samtools/idxstats" tag "samtools/flagstat" tag "samtools/stats" tag "bam_sort_stats_samtools" tag "fastq_align_bwa" tag "picard/markduplicates" tag "picard/addorreplacereadgroups" - tag "untar" setup { run("BWA_INDEX") { @@ -107,7 +111,6 @@ nextflow_workflow { then { assertAll( - { assert workflow.success }, { assert workflow.success }, { assert snapshot( workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, @@ -153,7 +156,6 @@ nextflow_workflow { then { assertAll( - { assert workflow.success }, { assert workflow.success }, { assert snapshot( workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, @@ -200,17 +202,10 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, - workflow.out.bai.collect { meta, bai -> file(bai).name }, - workflow.out.samtools_flagstat, - workflow.out.samtools_stats, - workflow.out.samtools_index_stats, - workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, - workflow.out.multiqc.flatten().collect { path -> file(path).name }, - workflow.out.versions - ).match() } + workflow.out, + workflow.out.versions.collect{ path(it).yaml } + ).match() } ) } } - } From 3378759b73906734188566e2fc4f7547802375f1 Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Mon, 3 Nov 2025 11:06:50 -0700 Subject: [PATCH 17/21] PR feedback --- .../tests/gpu.nf.test | 4 +++ .../tests/main.nf.test | 36 ++++++++++--------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index e897492d947d..8a76c859ca28 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -60,6 +60,7 @@ nextflow_workflow { input[5] = true // use_gpu input[6] = [[:], [] ] // interval_file input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } @@ -103,6 +104,7 @@ nextflow_workflow { input[5] = true // use_gpu input[6] = [[:], [] ] // interval_file input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } @@ -150,6 +152,7 @@ nextflow_workflow { input[5] = true // use_gpu input[6] = [[:], [] ] // interval_file input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } @@ -194,6 +197,7 @@ nextflow_workflow { input[5] = true // use_gpu input[6] = [[:], [] ] // interval_file input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test index eb4ea0caac28..2e8bcd4d58ed 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test @@ -56,10 +56,11 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = false // skip_deduplication - input[5] = false // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = false // skip_deduplication + input[5] = false // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } @@ -101,10 +102,11 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = false // skip_deduplication - input[5] = false // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = false // skip_deduplication + input[5] = false // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } @@ -146,10 +148,11 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = true // skip_deduplication - input[5] = false // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = true // skip_deduplication + input[5] = false // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } @@ -190,10 +193,11 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) ]) input[3] = BWA_INDEX.out.index - input[4] = false // skip_deduplication - input[5] = false // use_gpu - input[6] = Channel.empty() // interval_file - input[7] = Channel.empty() // known_sites + input[4] = false // skip_deduplication + input[5] = false // use_gpu + input[6] = [[:], [] ] // interval_file + input[7] = [[:], [] ] // known_sites + input[8] = "bam" // output_fmt """ } } From 54d4284495a592beb74cc79ec7c5e9df4c0c9bd0 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 3 Nov 2025 18:11:22 +0000 Subject: [PATCH 18/21] new cpu snapshot --- .../tests/main.nf.test.snap | 291 +++++++++++++----- 1 file changed, 220 insertions(+), 71 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap index ae1573a72264..0c41744cc233 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/main.nf.test.snap @@ -78,7 +78,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-31T15:27:53.091677319" + "timestamp": "2025-11-03T18:07:22.444845091" }, "Params: bwamem paired-end - skip_deduplication": { "content": [ @@ -156,88 +156,237 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-31T15:28:18.193321062" + "timestamp": "2025-11-03T18:07:48.375738382" }, "Params: bwamem single-end - default - stub": { "content": [ - [ - "d41d8cd98f00b204e9800998ecf8427e" - ], - [ - "test.deduped.sorted.bam.bai" - ], - [ - [ - { - "id": "test", - "single_end": true - }, - "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - [ - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], - [ - { - "id": "test", - "single_end": true - }, - "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - [ - [ - { - "id": "test", - "single_end": true - }, - "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + "2": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ], + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], - [ - { - "id": "test", - "single_end": true - }, - "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + "3": [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "5": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "6": [ + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ], + [ + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "7": [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,8f2ff4c430c4c5f42631534eaf358833", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + ], + "bai": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "multiqc": [ + [ + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ], + [ + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "picard_metrics": [ + [ + { + "id": "test", + "single_end": true + }, + "test.deduped.sorted.MarkDuplicates.metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "samtools_flagstat": [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,67394650dbae96d1a4fcc70484822159" + ], + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "samtools_index_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.idxstats:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "samtools_stats": [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,6fa0c192669339220d5c5735739188ac", + "versions.yml:md5,8f2ff4c430c4c5f42631534eaf358833", + "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", + "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" ] - ], - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt" - ], - [ - "test.deduped.sorted.MarkDuplicates.metrics.txt", - "test.sorted.bam", - "test.flagstat", - "test.sorted.bam", - "test.idxstats", - "test.sorted.bam", - "test.stats" - ], - [ - "versions.yml:md5,6fa0c192669339220d5c5735739188ac", - "versions.yml:md5,8f2ff4c430c4c5f42631534eaf358833", - "versions.yml:md5,c8ad9ea35566f100e41588f4c271b663", - "versions.yml:md5,e0c30eb274e33f3086ff9e694aae2161" + }, + [ + { + "FASTQ_ALIGN_DEDUP_BWAMEM:PICARD_MARKDUPLICATES": { + "picard": "3.4.0" + } + }, + { + "FASTQ_ALIGN_DEDUP_BWAMEM:FASTQ_ALIGN_BWA:BWA_MEM": { + "bwa": "0.7.19-r1273", + "samtools": "1.22.1" + } + }, + { + "FASTQ_ALIGN_DEDUP_BWAMEM:SAMTOOLS_INDEX": { + "samtools": "1.22.1" + } + }, + { + "FASTQ_ALIGN_DEDUP_BWAMEM:PICARD_ADDORREPLACEREADGROUPS": { + "picard": "3.4.0" + } + } ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-31T15:28:31.607014969" + "timestamp": "2025-11-03T18:08:01.772454395" }, "Params: bwamem paired-end - default": { "content": [ @@ -318,6 +467,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.5" }, - "timestamp": "2025-10-31T15:28:08.721629597" + "timestamp": "2025-11-03T18:07:38.589146631" } } \ No newline at end of file From 3cd511d989044a543d03c58817147037725c74de Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Mon, 3 Nov 2025 11:24:13 -0700 Subject: [PATCH 19/21] changing test --- .../nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index 8a76c859ca28..671bfcd1a3e2 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -30,8 +30,7 @@ nextflow_workflow { script "../../../../modules/nf-core/bwa/index/main.nf" process { """ - input[0] = Channel.of([ - [ id:'test' ], // meta map + input[0] = Channel.value([ [ id:'genome' ], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) """ From 1682b6b0bad0ef081d8823922b74ec32b41310db Mon Sep 17 00:00:00 2001 From: eduard-watchmaker Date: Mon, 3 Nov 2025 11:29:55 -0700 Subject: [PATCH 20/21] changing test --- .../nf-core/fastq_align_dedup_bwamem/main.nf | 12 ++++++------ .../fastq_align_dedup_bwamem/tests/gpu.nf.test | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf index 6dae11abacda..f11385d50906 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/main.nf @@ -33,12 +33,12 @@ workflow FASTQ_ALIGN_DEDUP_BWAMEM { * Align with parabricks GPU enabled fq2bammeth implementation of bwameth */ PARABRICKS_FQ2BAM ( - ch_reads, - ch_fasta, - ch_bwamem_index, - interval_file, // interval file - known_sites, // known sites - output_fmt // output format + ch_reads, // channel: [ val(meta), [ reads ] ] + ch_fasta, // channel: [ val(meta), [ fasta ] ] + ch_bwamem_index, // channel: [ val(meta), [ bwamem index ] ] + interval_file, // channel: [ val(meta), [ interval file ] ] + known_sites, // channel: [ val(meta), [ known sites ] ] + output_fmt // string: output format ) ch_alignment = PARABRICKS_FQ2BAM.out.bam ch_versions = ch_versions.mix(PARABRICKS_FQ2BAM.out.versions.first()) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test index 671bfcd1a3e2..c858a5be3d98 100755 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test @@ -57,8 +57,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // deduplicate input[5] = true // use_gpu - input[6] = [[:], [] ] // interval_file - input[7] = [[:], [] ] // known_sites + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites input[8] = "bam" // output_fmt """ } @@ -101,8 +101,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = true // skip_deduplication input[5] = true // use_gpu - input[6] = [[:], [] ] // interval_file - input[7] = [[:], [] ] // known_sites + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites input[8] = "bam" // output_fmt """ } @@ -149,8 +149,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = true // skip_deduplication input[5] = true // use_gpu - input[6] = [[:], [] ] // interval_file - input[7] = [[:], [] ] // known_sites + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites input[8] = "bam" // output_fmt """ } @@ -194,8 +194,8 @@ nextflow_workflow { input[3] = BWA_INDEX.out.index input[4] = false // deduplicate input[5] = true // use_gpu - input[6] = [[:], [] ] // interval_file - input[7] = [[:], [] ] // known_sites + input[6] = Channel.empty() // interval_file + input[7] = Channel.empty() // known_sites input[8] = "bam" // output_fmt """ } From a1b02e2e5225941da8729f0b6b3f97a16ac1d5fc Mon Sep 17 00:00:00 2001 From: root Date: Mon, 3 Nov 2025 18:30:52 +0000 Subject: [PATCH 21/21] new gpu snapshot --- .../tests/gpu.nf.test.snap | 133 +++++++++--------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap index 8c85b6b0f6a7..0d60654df785 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bwamem/tests/gpu.nf.test.snap @@ -1,64 +1,5 @@ { - "Params: parabricks/fq2bam single-end | use_gpu | stub": { - "content": [ - { - "0": [ - - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - - ], - "4": [ - - ], - "5": [ - - ], - "6": [ - - ], - "7": [ - - ], - "bai": [ - - ], - "bam": [ - - ], - "multiqc": [ - - ], - "picard_metrics": [ - - ], - "samtools_flagstat": [ - - ], - "samtools_index_stats": [ - - ], - "samtools_stats": [ - - ], - "versions": [ - - ] - } - ], - "meta": { - "nf-test": "0.9.3", - "nextflow": "25.04.7" - }, - "timestamp": "2025-10-31T15:58:31.925828509" - }, - "Params: parabricks/fq2bam single-end | use_gpu ": { + "Sarscov2 fasta - SE - deduplicate - with GPU parabricks/fq2bam": { "content": [ [ @@ -89,9 +30,9 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:58:17.076913842" + "timestamp": "2025-11-03T18:13:59.590746985" }, - "Params: parabricks/fq2bam paired-end | use_gpu | skip_deduplication": { + "Sarscov2 fasta - PE - skip deduplication - with GPU parabricks/fq2bam": { "content": [ [ @@ -122,9 +63,71 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:58:39.347884795" + "timestamp": "2025-11-03T18:14:13.651391069" + }, + "Sarscov2 fasta - SE - skip deduplication - with GPU parabricks/fq2bam - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "bai": [ + + ], + "bam": [ + + ], + "multiqc": [ + + ], + "picard_metrics": [ + + ], + "samtools_flagstat": [ + + ], + "samtools_index_stats": [ + + ], + "samtools_stats": [ + + ], + "versions": [ + + ] + }, + [ + + ] + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.04.7" + }, + "timestamp": "2025-11-03T18:14:20.925592761" }, - "Params: parabricks/fq2bam single-end | use_gpu | skip_deduplication": { + "Sarscov2 fasta - SE - skip deduplication - with GPU parabricks/fq2bam": { "content": [ [ @@ -155,6 +158,6 @@ "nf-test": "0.9.3", "nextflow": "25.04.7" }, - "timestamp": "2025-10-31T15:58:24.519410831" + "timestamp": "2025-11-03T18:14:06.680381689" } } \ No newline at end of file