Dev -> Main for release 0.7.0

.devcontainer/devcontainer.json

@@ -1,20 +1,32 @@
+//
+// MAIN PRODUCTION DEVCONTAINER CONFIG
+// Uses image that is pre-built and pushed to GitHub
+// See .github/.devcontainer/devcontainer.json for build
+//
 {
-    "name": "nfcore",
-    "image": "nfcore/gitpod:latest",
-    "remoteUser": "gitpod",
-    "runArgs": ["--privileged"],
-
-    // Configure tool-specific properties.
+    "name": "nf-aggregate",
+    "image": "ghcr.io/nextflow-io/training:latest",
+    "workspaceFolder": "/workspaces/nf-aggregate",
+    "remoteUser": "root",
+    "remoteEnv": {
+        // Nextflow installation version
+        "NXF_HOME": "/workspaces/.nextflow",
+        "NXF_EDGE": "0",
+        "NXF_VER": "24.10.4",
+        // Other env vars
+        "HOST_PROJECT_PATH": "/workspaces/nf-aggregate",
+        "SHELL": "/bin/bash" // Ush bash
+    },
+    "onCreateCommand": "bash .devcontainer/setup.sh",
     "customizations": {
-        // Configure properties specific to VS Code.
         "vscode": {
-            // Set *default* container specific settings.json values on container create.
+            "extensions": ["nf-core.nf-core-extensionpack"],
+            // Use Python from conda
             "settings": {
                 "python.defaultInterpreterPath": "/opt/conda/bin/python"
             },
-
-            // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
+            // Use bash
+            "terminal.integrated.defaultProfile.linux": "bash"
         }
     }
 }

.devcontainer/setup.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Fix for Java options
+printf 'unset JAVA_TOOL_OPTIONS\n' >> $HOME/.bashrc
+unset JAVA_TOOL_OPTIONS
+
+# Customise the terminal command prompt
+printf "export PS1='\\[\\e[3;36m\\]\${PWD#/workspaces/} ->\\[\\e[0m\\] '\n" >> $HOME/.bashrc
+export PS1='\[\e[3;36m\]${PWD#/workspaces/} ->\[\e[0m\] '
+
+# Force Java environment variables to use conda installation
+printf 'export JAVA_HOME=/opt/conda\nexport JAVA_CMD=/opt/conda/bin/java\n' >> $HOME/.bashrc
+export JAVA_HOME=/opt/conda
+export JAVA_CMD=/opt/conda/bin/java
+
+# Update Nextflow
+nextflow self-update
+nextflow -version
+
+cat /usr/local/etc/vscode-dev-containers/first-run-notice.txt

.nf-core.yml

@@ -58,4 +58,4 @@ template:
     - ci
     - igenomes
     - nf_core_configs
-  version: 0.6.0
+  version: 0.7.0

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "markdown.styles": ["public/vscode_markdown.css"]
+    "markdown.styles": ["public/vscode_markdown.css"],
+    "nextflow.telemetry.enabled": false
 }

CHANGELOG.md

@@ -3,7 +3,36 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## 0.6.0
+## [[0.7.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.7.0)] - 2025-05-05
+
+Special thanks to the following for their contributions to the release:
+
+- [Edmund Miller](https://github.com/edmundmiller)
+- [Florian Wuennemann](https://github.com/FloWuenne)
+- [Maxime Garcia](https://github.com/maxulysse)
+
+Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
+
+### Enhancements & fixes
+
+- [PR #88](https://github.com/seqeralabs/nf-aggregate/pull/88) - Update tw cli container version to 0.11.2 and allow .nextflow.log to be missing from tw call
+- [PR #89](https://github.com/seqeralabs/nf-aggregate/pull/89) - Enable usage of external run dumps with nf-aggregate & update devcontainer specifications
+- [PR #91](https://github.com/seqeralabs/nf-aggregate/pull/91) - Update benchmark report image to include a fix causing large memory footprint for reshaping large AWS cost report files
+
+### Software dependencies
+
+| Dependency        | Old version | New version |
+| ----------------- | ----------- | ----------- |
+| `tower-cli`       | 0.9.2       | 0.11.2      |
+| BENCHMARK_REPORTS | sha-48cfed7 | sha-a6d15e8 |
+
+> **NB:** Dependency has been **updated** if both old and new version information is present.
+>
+> **NB:** Dependency has been **added** if just the new version information is present.
+>
+> **NB:** Dependency has been **removed** if new version information isn't present.
+
+## [[0.6.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.6.0)] - 2025-03-31
 
 ### Credits
 

README.md

@@ -69,6 +69,21 @@ id,workspace,group
 4VLRs7nuqbAhDy,community/showcase,group2
 ```
 
+## Use logs from an external Seqera Platform deployment
+
+Sometimes we want to compile benchmark reports from runs from two different Seqera platform deployments, for example a dev and a production environment to compare performance. External logs in nf-aggregate can be used by specifying the workspace as `external` and providing some additional optional columns that point to the log folder and specify whether these external logs contain fusion logs (did you export them with the `--add-fusion-logs` flag in your `tw run dumps`. If they do contain fusion logs, you can generate a gannt plot for them, as for runs supplied only via id.)
+
+Here is an example of using a mix of run ids for which we want to extract logs from our platform deployment and some run logs from another deployment we want to compare. In the example below, `1JI5B1avuj3o58` is a run that contains fusion logs, while `1vsww7GjKBsVNa` does not contain fusion logs.
+
+```
+id,workspace,group,logs,fusion
+3VcLMAI8wyy0Ld,community/showcase,group1,
+1JI5B1avuj3o58,external,group2,/path/to/my/run_dumps_tarball.tar.gz,true
+1vsww7GjKBsVNa,external,group2,/path/to/my/run_dumps_folder,false
+```
+
+## Incorporate AWS split cost allocation data
+
 To incorporate AWS cost data into the benchmark report, use the `benchmark_aws_cur_report` parameter. This should point to a valid AWS Cost and Usage Report (CUR) file in Parquet format, currently only supporting CUR 1.0. The file can be stored locally or in a cloud bucket.
 To run nf-aggregate and generate benchmark reports, you can use the following command:
 

assets/schema_input.json

@@ -15,15 +15,26 @@
             },
             "workspace": {
                 "type": "string",
-                "pattern": "^[a-zA-Z0-9](?:[a-zA-Z0-9]|[-_](?=[a-zA-Z0-9])){1,38}/[a-zA-Z0-9](?:[a-zA-Z0-9]|[-_](?=[a-zA-Z0-9])){1,38}$",
-                "errorMessage": "Please provide a valid Seqera Platform Workspace name",
+                "pattern": "^(?:external|[a-zA-Z0-9][a-zA-Z0-9-_]{0,38}/[a-zA-Z0-9][a-zA-Z0-9-_]{0,38})$",
+                "errorMessage": "Please provide a valid Seqera Platform Workspace name or 'external'",
                 "meta": ["workspace"]
             },
             "group": {
                 "type": "string",
                 "pattern": "^[a-zA-Z0-9][-a-zA-Z0-9_ ]{0,37}$",
                 "errorMessage": "Please provide a valid group name",
                 "meta": ["group"]
+            },
+            "logs": {
+                "type": "string",
+                "format": "file-path",
+                "pattern": "^\\S+$",
+                "errorMessage": "Please provide a valid file path to your Seqera Platform logs.",
+                "meta": ["logs"]
+            },
+            "fusion": {
+                "type": "boolean",
+                "meta": ["fusion"]
             }
         },
         "required": ["id", "workspace"]

modules/local/benchmark_report/main.nf

@@ -1,6 +1,6 @@
 process BENCHMARK_REPORT {
 
-    container 'cr.seqera.io/scidev/benchmark-reports:sha-48cfed7'
+    container 'cr.seqera.io/scidev/benchmark-reports:sha-a6d15e8'
 
     input:
     path run_dumps

modules/local/benchmark_report/nextflow.config

@@ -2,7 +2,7 @@ process {
     withName: 'BENCHMARK_REPORT' {
         publishDir = [
             path: { "${params.outdir}/${metaOut?.projectName?.replace("/", "_") ?: ""}/benchmark_report" },
-            mode: params.publish_dir_mode,
+            mode: 'copy',
             saveAs: { filename -> filename.equals('versions.yml') || filename.endsWith('.json') ? null : filename }
         ]
         containerOptions = "--user root"

modules/local/plot_run_gantt/nextflow.config

@@ -2,7 +2,7 @@ process {
     withName: 'PLOT_RUN_GANTT' {
         publishDir = [
             path: { "${params.outdir}/${meta.projectName?.replace("/", "_") ?: ""}/gantt" },
-            mode: params.publish_dir_mode,
+            mode: 'copy',
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }

modules/local/seqera_runs_dump/main.nf

@@ -3,8 +3,8 @@ include { getRunMetadata } from './functions'
 process SEQERA_RUNS_DUMP {
     tag "${meta.id}"
 
-    conda 'tower-cli=0.9.2'
-    container 'seqeralabs/nf-aggregate:tower-cli-0.9.2--hdfd78af_1'
+    conda 'tower-cli=0.11.2'
+    container 'community.wave.seqera.io/library/tower-cli:0.11.2--0f5ebc1e8a308611'
 
     input:
     val meta
@@ -37,7 +37,7 @@ process SEQERA_RUNS_DUMP {
         --workspace=${meta.workspace} \\
         --output="${prefix}.tar.gz" \\
         ${fusion} \\
-        ${args2}
+        ${args2} 2>&1 || if [ ! -f "${prefix}.tar.gz" ]; then exit 1; fi
 
     mkdir -p ${prefix}
     tar \\

modules/local/seqera_runs_dump/nextflow.config

@@ -6,7 +6,7 @@ process {
         containerOptions = { params.java_truststore_path ? "--volume ${params.java_truststore_path}:${params.java_truststore_path}" : '' }
         publishDir = [
             path: { "${params.outdir}/${metaOut?.projectName?.replace("/", "_") ?: ""}/runs_dump" },
-            mode: params.publish_dir_mode,
+            mode: 'copy',
             saveAs: { filename -> filename.equals('versions.yml') || filename.endsWith('.json') ? null : filename }
         ]
     }

modules/local/seqera_runs_dump/tests/main.nf.test.snap

@@ -7,19 +7,19 @@
             ],
             [
                 "workflow-load.json:md5,4f02d5a24ab89aa648cd4346785c8f2c",
-                "workflow-metadata.json:md5,b37b4faeddf283a2c44cbe4000e4ab6e",
+                "workflow-metadata.json:md5,8f3466573bb85a7005975d7929687ab5",
                 "workflow-metrics.json:md5,13a5b5d7447fad4a8baa053d1abf85e5",
-                "workflow-tasks.json:md5,577a7472816b7729012a9291d97ff150",
-                "workflow.json:md5,658701f8d7af1b7112534a7f58cd9aa7"
+                "workflow-tasks.json:md5,e327f5aae611c7f731a470bd7390d967",
+                "workflow.json:md5,59f8a9ea713a57ea48470ff0f92cc7e5"
             ],
             [
-                "versions.yml:md5,0d0b37ceb8fe6841f375e91d47dfa554"
+                "versions.yml:md5,f4475d693b7a861c1f17c8f6e265dda0"
             ]
         ],
         "meta": {
             "nf-test": "0.9.2",
-            "nextflow": "25.02.1"
+            "nextflow": "25.01.0"
         },
-        "timestamp": "2025-03-28T16:40:15.32641466"
+        "timestamp": "2025-04-14T16:30:25.863976"
     }
 }

nextflow.config

@@ -62,6 +62,12 @@ profiles {
     test {
         params.input = "${projectDir}/workflows/nf_aggregate/assets/test_run_ids.csv"
     }
+    test_benchmark {
+        params.input                        = "${projectDir}/workflows/nf_aggregate/assets/test_benchmark.csv"
+        params.generate_benchmark_report    = true
+        params.skip_run_gantt               = true
+        params.skip_multiqc                 = true
+    }
     test_full {
         params.input = "${projectDir}/workflows/nf_aggregate/assets/test_run_ids.csv"
     }
@@ -314,13 +320,13 @@ manifest {
     mainScript      = 'main.nf'
     defaultBranch   = 'main'
     nextflowVersion = '!>=24.04.2'
-    version         = '0.6.0'
+    version         = '0.7.0'
     doi             = ''
 }
 
 // Nextflow plugins
 plugins {
-    id 'nf-schema@2.3.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet
+    id 'nf-schema@2.2.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet
 }
 
 validation {