Merge pull request #1025 from drpatelh/updates

Add public_aws_ecr.config and use in CI tests

Author: drpatelh

.github/workflows/cloud_tests_full.yml

@@ -30,7 +30,7 @@ jobs:
           compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
           workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/rnaseq/work-${{ github.sha }}"
           run_name: "aws_rnaseq_full_${{ matrix.aligner }}"
-          profiles: test_full_aws
+          profiles: test_full_aws,public_aws_ecr
           parameters: |
             {
                 "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",

.github/workflows/cloud_tests_small.yml

@@ -25,7 +25,7 @@ jobs:
           compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
           workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/rnaseq/work-${{ github.sha }}"
           run_name: "aws_rnaseq_small"
-          profiles: test
+          profiles: test,public_aws_ecr
           parameters: |
             {
                 "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/rnaseq/results-test-${{ github.sha }}"

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Enhancements & fixes
 
 - [PR #1016](https://github.com/nf-core/rnaseq/pull/1016) - Updated pipeline template to [nf-core/tools 2.8](https://github.com/nf-core/tools/releases/tag/2.8)
+- [PR #1025](https://github.com/nf-core/fetchngs/pull/1025) - Add `public_aws_ecr.config` to source mulled containers when using `public.ecr.aws` Docker Biocontainer registry
 
 ### Software dependencies
 

conf/public_aws_ecr.config

@@ -0,0 +1,54 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    AWS ECR Config
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Config to set public AWS ECR images wherever possible
+    This improves speed when running on AWS infrastructure.
+    Use this as an example template when using your own private registry.
+----------------------------------------------------------------------------------------
+*/
+
+docker.registry = 'public.ecr.aws'
+podman.registry = 'public.ecr.aws'
+
+process {
+    withName: '.*:CAT_FASTQ' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+    withName: '.*:DESEQ2_QC' {
+        container = 'quay.io/biocontainers/mulled-v2-8849acf39a43cdd6c839a369a74c0adc823e2f91:ab110436faf952a33575c64dd74615a84011450b-0'
+    }
+    withName: '.*:GUNZIP' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+    withName: '.*:HISAT2_ALIGN' {
+        container = 'quay.io/biocontainers/mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1:2cdf6bf1e92acbeb9b2834b1c58754167173a410-0'
+    }
+    withName: '.*:PREPROCESS_TRANSCRIPTS_FASTA_GENCODE' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+    withName: '.*:RSEM_CALCULATEEXPRESSION' {
+        container = 'quay.io/biocontainers/mulled-v2-cf0123ef83b3c38c13e3b0696a3f285d3f20f15b:64aad4a4e144878400649e71f42105311be7ed87-0'
+    }
+    withName: '.*:RSEM_MERGE_COUNTS' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+    withName: '.*:RSEM_PREPAREREFERENCE' {
+        container = 'quay.io/biocontainers/mulled-v2-cf0123ef83b3c38c13e3b0696a3f285d3f20f15b:64aad4a4e144878400649e71f42105311be7ed87-0'
+    }
+    withName: '.*:STAR_ALIGN' {
+        container = 'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0'
+    }
+    withName: '.*:STAR_ALIGN_IGENOMES' {
+        container = 'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0'
+    }
+    withName: '.*:STAR_GENOMEGENERATE' {
+        container = 'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0'
+    }
+    withName: '.*:STAR_GENOMEGENERATE_IGENOMES' {
+        container = 'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0'
+    }
+    withName: '.*:UNTAR' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+}

nextflow.config

@@ -230,9 +230,14 @@ profiles {
         executor.cpus          = 16
         executor.memory        = 60.GB
     }
+    public_aws_ecr {
+        includeConfig 'conf/public_aws_ecr.config'
+    }
     test          { includeConfig 'conf/test.config'      }
     test_full     { includeConfig 'conf/test_full.config' }
-    test_full_aws { includeConfig 'conf/test_full.config' }
+    test_full_aws { 
+        includeConfig 'conf/test_full.config' 
+    }
     test_full_gcp {
         includeConfig 'conf/test_full.config'
         params.input = 'https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/samplesheet/v3.10/samplesheet_full_gcp.csv'