From 2b727520ca9ed47913082beedc776f7e5096f0ed Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 29 Oct 2025 11:03:36 +0100 Subject: [PATCH 1/4] Add nf-co2footprint plugin for emissions tracking --- .github/workflows/nf-test.yml | 25 +++++++++++++++++++++++ modules/nf-core/shasta/tests/main.nf.test | 2 +- tests/config/nf-test.config | 4 ++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 97e55ba5c666..0cdd4f8a486c 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -171,6 +171,31 @@ jobs: total_shards: ${{ env.TOTAL_SHARDS }} paths: "${{ join(fromJson(steps.filter.outputs.filtered_paths), ' ') }}" + - name: Upload CO2 footprint reports to S3 + if: always() && steps.filter.outputs.filtered_paths != '[]' + shell: bash + run: | + # Find all CO2 footprint trace files + CO2_FILES=$(find . -name "co2footprint_trace*.txt" -type f) + + if [ -n "$CO2_FILES" ]; then + # Get current date and sanitize branch name + DATE=$(date +%Y-%m-%d) + BRANCH_NAME=$(echo "${{ github.head_ref || github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') + + # S3 destination path + S3_BASE="s3://nf-core-resource-optimization/co2reports/modules/${DATE}/${BRANCH_NAME}/${{ matrix.profile }}/${{ matrix.shard }}" + + # Upload each CO2 trace file + for file in $CO2_FILES; do + filename=$(basename "$file") + aws s3 cp "$file" "${S3_BASE}/${filename}" --no-progress + echo "Uploaded: $file -> ${S3_BASE}/${filename}" + done + else + echo "No CO2 footprint trace files found" + fi + confirm-pass-nf-test: runs-on: - runs-on=${{ github.run_id }}-confirm-pass-nf-test diff --git a/modules/nf-core/shasta/tests/main.nf.test b/modules/nf-core/shasta/tests/main.nf.test index ee6654930a7e..1d5c8760bb1a 100644 --- a/modules/nf-core/shasta/tests/main.nf.test +++ b/modules/nf-core/shasta/tests/main.nf.test @@ -1,4 +1,4 @@ - +// Trigger CI to test CO2 footprint tracking nextflow_process { name "Test Process SHASTA" diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 336f2e178917..2e5af5e2c499 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -1,3 +1,7 @@ +plugins { + id 'nf-co2footprint@1.0.0' +} + params { publish_dir_mode = "copy" singularity_pull_docker_container = false From c0a83d81ce93b89e3621206bbaeebc78e57dd47c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 29 Oct 2025 17:07:27 +0100 Subject: [PATCH 2/4] chore: update CO2 reports to use nf-core-co2-reports bucket --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 0cdd4f8a486c..230e1cc25a3a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -184,7 +184,7 @@ jobs: BRANCH_NAME=$(echo "${{ github.head_ref || github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') # S3 destination path - S3_BASE="s3://nf-core-resource-optimization/co2reports/modules/${DATE}/${BRANCH_NAME}/${{ matrix.profile }}/${{ matrix.shard }}" + S3_BASE="s3://nf-core-co2-reports/modules/${DATE}/${BRANCH_NAME}/${{ matrix.profile }}/${{ matrix.shard }}" # Upload each CO2 trace file for file in $CO2_FILES; do From 64587e772f8d7d25ee7e44084c5c6f1989a9bb2a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 2 Nov 2025 11:36:14 +0100 Subject: [PATCH 3/4] ci: add AWS credentials for CO2 reports S3 upload --- .github/workflows/nf-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 230e1cc25a3a..020464b475a7 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -171,6 +171,14 @@ jobs: total_shards: ${{ env.TOTAL_SHARDS }} paths: "${{ join(fromJson(steps.filter.outputs.filtered_paths), ' ') }}" + - name: Configure AWS credentials for CO2 reports + if: always() && steps.filter.outputs.filtered_paths != '[]' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.CO2_REPORTS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.CO2_REPORTS_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.CO2_REPORTS_AWS_REGION }} + - name: Upload CO2 footprint reports to S3 if: always() && steps.filter.outputs.filtered_paths != '[]' shell: bash From 73f1087d6a5965e755f0f539bcda066f52d9c8d3 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 2 Nov 2025 12:54:27 +0100 Subject: [PATCH 4/4] ci: fail CO2 upload step when no trace files found --- .github/workflows/nf-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 020464b475a7..2726a47ebf37 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -201,7 +201,8 @@ jobs: echo "Uploaded: $file -> ${S3_BASE}/${filename}" done else - echo "No CO2 footprint trace files found" + echo "ERROR: No CO2 footprint trace files found" + exit 1 fi confirm-pass-nf-test: