From 086f6096d148e8bac9459a69bae27841d077d394 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 13 Mar 2025 17:25:20 -0500 Subject: [PATCH 1/2] Use nf-boost standard library functions Signed-off-by: Ben Sherman --- .github/workflows/ci.yml | 2 +- nextflow.config | 3 +- .../nf-core/utils_nextflow_pipeline/main.nf | 12 ++--- .../nf-core/utils_nfcore_pipeline/main.nf | 49 ++++++++----------- 4 files changed, 28 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efd086d6..98411369 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: fail-fast: false matrix: NXF_VER: - - "24.04.2" + - "24.10.5" - "latest-everything" profile: - "conda" diff --git a/nextflow.config b/nextflow.config index a047ab96..0f4320f4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -220,7 +220,7 @@ manifest { homePage = 'https://github.com/nf-core/fetchngs' description = """Pipeline to fetch metadata and raw FastQ files from public databases""" mainScript = 'main.nf' - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=24.10.0' version = '1.13.0dev' doi = '10.5281/zenodo.5070524' } @@ -228,6 +228,7 @@ manifest { // Nextflow plugins plugins { id 'nf-schema@2.1.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-boost@0.5.0' } validation { diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index 0fcbf7b3..a938c923 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -1,6 +1,7 @@ // // Subworkflow with functionality that may be useful for any Nextflow pipeline // +include { fromYaml ; toJson } from 'plugin/nf-boost' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -72,12 +73,10 @@ def getWorkflowVersion() { // def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') - def filename = "params_${timestamp}.json" - def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = groovy.json.JsonOutput.toJson(params) - temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) + def temp_pf = file(".params_${timestamp}.json") + temp_pf.text = toJson(params, true) - nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.copyTo("${outdir}/pipeline_info/params_${timestamp}.json") temp_pf.delete() } @@ -85,10 +84,9 @@ def dumpParametersToJSON(outdir) { // When running with -profile conda, warn if channels have not been set-up appropriately // def checkCondaChannels() { - def parser = new org.yaml.snakeyaml.Yaml() def channels = [] try { - def config = parser.load("conda config --show channels".execute().text) + def config = fromYaml("conda config --show channels".execute().text) channels = config.channels } catch (NullPointerException e) { diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 5cb7bafe..aec3757b 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -1,6 +1,7 @@ // // Subworkflow with utility functions specific to the nf-core pipeline template // +include { fromYaml ; toYaml ; request ; template } from 'plugin/nf-boost' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -93,9 +94,8 @@ def getWorkflowVersion() { // Get software versions for pipeline // def processVersionsFromYAML(yaml_file) { - def yaml = new org.yaml.snakeyaml.Yaml() - def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } - return yaml.dumpAsMap(versions).trim() + def versions = fromYaml(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } + return toYaml(versions).trim() } // @@ -329,31 +329,28 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi } // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("${workflow.projectDir}/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() + def tf = file("${workflow.projectDir}/assets/email_template.txt") + def email_txt = template(tf, email_fields) // Render the HTML template - def hf = new File("${workflow.projectDir}/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() + def hf = file("${workflow.projectDir}/assets/email_template.html") + def email_html = template(hf, email_fields) // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] - def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() + def sf = file("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_html = template(sf, smail_fields) // Send the HTML e-mail def colors = logColours(monochrome_logs) as Map if (email_address) { try { if (plaintext_email) { -new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } + throw new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') + } // Try to send HTML e-mail using sendmail - def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + def sendmail_tf = file(".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } ['sendmail', '-t'].execute() << sendmail_html log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") @@ -367,15 +364,15 @@ new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } // Write summary e-mail HTML to a file - def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + def output_hf = file(".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") + output_hf.copyTo("${outdir}/pipeline_info/pipeline_report.html") output_hf.delete() // Write summary e-mail TXT to a file - def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + def output_tf = file(".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") + output_tf.copyTo("${outdir}/pipeline_info/pipeline_report.txt") output_tf.delete() } @@ -441,20 +438,14 @@ def imNotification(summary_params, hook_url) { msg_fields['summary'] = summary << misc_fields // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() // Different JSON depending on the service provider // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() + def hf = file("${workflow.projectDir}/assets/${json_path}") + def json_message = template(hf, msg_fields) // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) + def post = request(hook_url, method: "POST", headers: ["Content-Type": "application/json"], body: json_message) def postRC = post.getResponseCode() if (!postRC.equals(200)) { log.warn(post.getErrorStream().getText()) From 900588a391b76d767c17f9e7acd7f031942cadb5 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 19 Mar 2025 14:47:09 -0500 Subject: [PATCH 2/2] Simplify file writes Signed-off-by: Ben Sherman --- .../nf-core/utils_nextflow_pipeline/main.nf | 7 ++----- subworkflows/nf-core/utils_nfcore_pipeline/main.nf | 13 +++---------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index a938c923..7f83e948 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -73,11 +73,8 @@ def getWorkflowVersion() { // def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') - def temp_pf = file(".params_${timestamp}.json") - temp_pf.text = toJson(params, true) - - temp_pf.copyTo("${outdir}/pipeline_info/params_${timestamp}.json") - temp_pf.delete() + def params_file = file("${outdir}/pipeline_info/params_${timestamp}.json") + params_file.text = toJson(params, true) } // diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index aec3757b..7f97c027 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -350,8 +350,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi throw new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail - def sendmail_tf = file(".sendmail_tmp.html") - sendmail_tf.withWriter { w -> w << sendmail_html } + file(".sendmail_tmp.html").text = sendmail_html ['sendmail', '-t'].execute() << sendmail_html log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") } @@ -364,16 +363,10 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi } // Write summary e-mail HTML to a file - def output_hf = file(".pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - output_hf.copyTo("${outdir}/pipeline_info/pipeline_report.html") - output_hf.delete() + file("${outdir}/pipeline_info/pipeline_report.html").text = email_html // Write summary e-mail TXT to a file - def output_tf = file(".pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - output_tf.copyTo("${outdir}/pipeline_info/pipeline_report.txt") - output_tf.delete() + file("${outdir}/pipeline_info/pipeline_report.txt").text = email_txt } //