diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb315423b..6dcba5fb4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,6 +90,7 @@ jobs: - "hasta" - "hki" - "hypatia" + - "icr_alma" - "icr_davros" - "ifb_core" - "imb" @@ -138,6 +139,7 @@ jobs: - "sanger" - "scw" - "seadragon" + - "seattlechildrens" - "seawulf" - "seg_globe" - "self_hosted_runner" diff --git a/README.md b/README.md index b214207af..c98604a84 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ Currently documentation is available for the following systems: - [HKI](docs/hki.md) - [HYPATIA](docs/hypatia.md) - [ICR_DAVROS](docs/icr_davros.md) +- [ICR_ALMA](docs/icr_alma.md) - [IFB](docs/ifb_core.md) - [ILIFU](docs/ilifu.md) - [IMPERIAL](docs/imperial.md) diff --git a/conf/engaging.config b/conf/engaging.config index 083cdedc7..246dfcea7 100644 --- a/conf/engaging.config +++ b/conf/engaging.config @@ -1,7 +1,7 @@ // Nextflow config for running on the MIT Engaging HPC cluster params { config_profile_description = 'MIT Engaging HPC cluster profile.' - config_profile_contact = 'Phil Palmer (@PhilPalmer)' + config_profile_contact = 'Charlie Whittaker (@bumproo)' config_profile_url = "https://engaging-web.mit.edu/eofe-wiki/" } @@ -12,16 +12,20 @@ singularity { process { resourceLimits = [ - memory: 64.GB, + memory: 128.GB, cpus: 16, - time: 12.h + time: 10.h ] executor = 'slurm' - clusterOptions = '-p sched_mit_hill' + clusterOptions = '-p mit_normal' +} + +executor { + queueSize = 8 } params { - max_memory = 64.GB + max_memory = 128.GB max_cpus = 16 - max_time = 12.h + max_time = 10.h } diff --git a/conf/icr_alma.config b/conf/icr_alma.config new file mode 100644 index 000000000..4cf2dabaa --- /dev/null +++ b/conf/icr_alma.config @@ -0,0 +1,59 @@ + +/* + * ------------------------------------------------- + * Nextflow nf-core config file for ICR alma HPC + * ------------------------------------------------- + * Defines slurm process executor and singularity + * settings. + * + */ +params { + + config_profile_description = "Nextflow nf-core profile for ICR alma HPC" + config_profile_contact = "Rachel Alcraft (@rachelicr), Mira Sarkis (@msarkis-icr)" + // max_memory = 256.GB + // max_cpus = 30 + // max_time = 5.d +} + +process { + queue="compute" + executor = "slurm" + maxRetries = 3 + maxErrors = '-1' + + errorStrategy = { task.exitStatus in [137,255] ? 'retry' : 'terminate' } + withName: ".*" { time = 5.d } + + clusterOptions = '--mem-per-cpu=8000' + + resourceLimits = [ + memory: 256.GB, + cpus: 30, + time: 5.d + ] + +} +// Preform work directory cleanup after a successful run? +cleanup = false + +executor { + // This is set because of an issue with too many + // singularity containers launching at once, they + // cause an singularity error with exit code 255. + // submitRateLimit = "2 sec" + queueSize = 50 + perCpuMemAllocation = true +} + +singularity { + enabled = true + // runOptions = "--bind /mnt:/mnt --bind /data:/data" + autoMounts = true + // pullTimeout = 2.h + // cacheDir = '/data/scratch/shared/SINGULARITY-DOWNLOAD/nextflow/.singularity' +} + + + + diff --git a/conf/ki_luria.config b/conf/ki_luria.config index d348ccd3e..f7a96439c 100644 --- a/conf/ki_luria.config +++ b/conf/ki_luria.config @@ -1,7 +1,7 @@ // Nextflow config for running on the Koch Institute of MIT Luria HPC cluster params { config_profile_description = 'KI at MIT Luria HPC cluster profile.' - config_profile_contact = 'Charlie Whittaker (@Charlie14557807)' + config_profile_contact = 'Charlie Whittaker (@bumproo)' config_profile_url = "https://igb.mit.edu/computing-resources/luria-cluster" } diff --git a/conf/mjolnir_globe.config b/conf/mjolnir_globe.config index a430cf3ee..f14147909 100644 --- a/conf/mjolnir_globe.config +++ b/conf/mjolnir_globe.config @@ -4,20 +4,19 @@ params { config_profile_contact = 'Bent Petersen (@bentpetersendk)' config_profile_url = 'https://globe.ku.dk/research/' max_memory = 750.GB - max_cpus = 50 + max_cpus = 48 max_time = 336.h } singularity { enabled = true autoMounts = true - cacheDir = '/maps/projects/mjolnir1/data/cache/nf-core/singularity' } process { resourceLimits = [ memory: 750.GB, - cpus: 50, + cpus: 48, time: 336.h ] executor = 'slurm' @@ -26,5 +25,5 @@ process { cleanup = true executor { - queueSize = 10 + queueSize = 20 } diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 632eb2224..d96b1306e 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -198,6 +198,11 @@ process { errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } + withName: genotyping_hc { + clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } + } + withName: get_software_versions { cache = false clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" } diff --git a/conf/roslin.config b/conf/roslin.config index d61c4f8a3..31cdd68b4 100644 --- a/conf/roslin.config +++ b/conf/roslin.config @@ -3,16 +3,18 @@ params { config_profile_description = 'University of Edinburgh (Eddie) cluster profile for Roslin Institute provided by nf-core/configs.' config_profile_contact = 'Sebastien Guizard (@sguizard) and Donald Dunbar (@ddunbar)' config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' + } executor { name = "sge" + } process { stageInMode = 'symlink' scratch = 'false' - penv = { task.cpus > 1 ? "sharedmem" : null } + penv = { task.cpus > 1 ? "sharedmem" : null } // To date (16/08/2024), the FastQC module is still broken. // More details here: https://github.com/nf-core/modules/pull/6156 @@ -24,9 +26,9 @@ process { // Check if an environment variable NFX_SGE_PROJECT exists, if yes, use the stored value for -P option // Otherwise set the project to uoe_baseline if (System.getenv('NFX_SGE_PROJECT')) { - clusterOptions = {"-l rl9=false -l h=!node1d01 -l h_vmem=10G -pe sharedmem 5 -P $NFX_SGE_PROJECT"} + clusterOptions = {"-l h=!node1d01 -l h_vmem=10G -pe sharedmem 5 -P $NFX_SGE_PROJECT"} } else { - clusterOptions = {"-l rl9=false -l h=!node1d01 -l h_vmem=10G -pe sharedmem 5 -P uoe_baseline"} + clusterOptions = {"-l h=!node1d01 -l h_vmem=10G -pe sharedmem 5 -P uoe_baseline"} } } @@ -41,9 +43,9 @@ process { // Check if an environment variable NFX_SGE_PROJECT exists, if yes, use the stored value for -P option // Otherwise set the project to uoe_baseline if (System.getenv('NFX_SGE_PROJECT')) { - clusterOptions = {"-l rl9=false -l h=!node1d01 -l h_vmem=${(task.memory + 8.GB).bytes/task.cpus} -P $NFX_SGE_PROJECT"} + clusterOptions = {"-l h=!node1d01 -l h_vmem=${(task.memory + 8.GB).bytes/task.cpus} -P $NFX_SGE_PROJECT"} } else { - clusterOptions = {"-l rl9=false -l h=!node1d01 -l h_vmem=${(task.memory + 8.GB).bytes/task.cpus} -P uoe_baseline"} + clusterOptions = {"-l h=!node1d01 -l h_vmem=${(task.memory + 8.GB).bytes/task.cpus} -P uoe_baseline"} } } @@ -76,3 +78,4 @@ singularity { autoMounts = true cacheDir = '/exports/cmvm/eddie/eb/groups/alaw3_eb_singularity_cache' } + diff --git a/conf/seadragon.config b/conf/seadragon.config index e92de492b..074bfb73f 100644 --- a/conf/seadragon.config +++ b/conf/seadragon.config @@ -24,33 +24,16 @@ singularity { def membership = "groups".execute().text def select_queue = { memory, cpu, walltime -> - // Cdragon queues - if (memory <= 168.GB && cpu <= 28) { + if (memory <= 950.GB && cpu <= 80) { if (walltime <= 3.h) return 'short' if (walltime <= 24.h) return 'medium' if (walltime <= 240.h) return 'long' - } - - // Edragon E40 queues - if (memory <= 475.GB && cpu <= 40) { - if (walltime <= 3.h) return 'e40short' - if (walltime <= 24.h) return 'e40medium' - if (walltime <= 240.h) return 'e40long' - } - - // Edragon E80 queues - if (memory <= 950.GB && cpu <= 80) { - if (walltime <= 3.h) return 'e80short' - if (walltime <= 24.h) return 'e80medium' - if (walltime <= 240.h) return 'e80long' + if (walltime <= 504.h) return 'vlong' } // High memory queues - if (memory <= 1900.GB && cpu <= 35) { - if (walltime <= 240.h) return 'highmem' - } - if (memory <= 2900.GB && cpu <= 24) { - if (walltime <= 240.h) return 'vhighmem' + if (memory <= 3900.GB && cpu <= 80 && walltime <= 504.h) { + if (walltime <= 240.h) return 'evhighmem' } throw new IllegalArgumentException("No matching queue for memory=${memory}, cpu=${cpu}, time=${time}") @@ -72,24 +55,21 @@ executor { process { resourceLimits = [ - memory: 2900.GB, // Max memory based on vhighmem node + memory: 3900.GB, // Max memory based on vhighmem node cpus: 80, // Max CPUs based on E80 node - time: 240.h // Max time for long queues + time: 504.h // Max time for long queues ] executor = 'lsf' // Use LSF executor - memory = { task.memory ?: params.default_memory } - cpus = { task.cpus ?: params.default_cpus } - time = { task.time ?: params.default_time } - + cpus = { 2 * task.attempt } + memory = { 12.GB * task.attempt } + time = { 3.h * task.attempt } maxRetries = 3 afterScript = 'sleep 10' // Prevent abrupt re-submissions after retries - queue = { select_queue(task.memory, task.cpus, task.time) } // Use the updated select_queue function - withLabel:process_gpu { cpus = { 40 } // Use Gdragon nodes memory = { 168.GB } // Max memory for GPU nodes @@ -98,9 +78,9 @@ process { } params { - max_memory = 2900.GB // Maximum memory based on vhighmem nodes + max_memory = 3900.GB // Maximum memory based on evhighmem nodes max_cpus = 80 // Maximum CPUs based on E80 nodes - max_time = 240.h // Maximum runtime for long queues + max_time = 504.h // Maximum runtime for evlong queues igenomes_base = '/rsrch3/scratch/reflib/REFLIB_data/AWS-iGenomes' } diff --git a/conf/seattlechildrens.config b/conf/seattlechildrens.config index 521d7b38f..455251362 100644 --- a/conf/seattlechildrens.config +++ b/conf/seattlechildrens.config @@ -1,30 +1,33 @@ -//Create profiles to easily switch between the different process executors and platforms. +def assoc = System.getenv("ASSOC") // Association belonging to a lab or project //global parameters params { config_profile_description = 'The SCRI (seattle childrens research institute) cluster profile' config_profile_contact = 'Research Scientific Computing (@RSC-RP)' - config_profile_url = 'https://github.com/RSC-RP' + config_profile_url = 'https://github.com/RSC-RP/nextflow_scri_config' +} + +//workDir = "/data/hps/assoc/private/${assoc}/user/$USER/temp" - // SCRI HPC project params - queue = "paidq" - // freeq - project = "${params.project}" +process { + executor = 'slurm' + queue = 'cpu-core-sponsored' + memory = 7500.MB + time = '72h' + clusterOptions = "--account cpu-${assoc}-sponsored" } +docker { + enabled = false +} + +singularity { + enabled = true + autoMounts = true + cacheDir = "/data/hps/assoc/private/${assoc}/container" + runOptions = '--containall --no-home' +} -profiles { - //For running on an interactive session on cybertron with singularity module loaded - local_singularity { - process.executor = 'local' - singularity.enabled = true - } - //For executing the jobs on the HPC cluster with singularity containers - PBS_singularity { - process.executor = 'pbspro' - process.queue = "${params.queue}" - process.clusterOptions = "-P ${params.project}" - process.beforeScript = 'module load singularity' - singularity.enabled = true - } +executor { + queueSize = 2000 } diff --git a/conf/unibe_ibu.config b/conf/unibe_ibu.config index c088b3bc1..1822785b4 100644 --- a/conf/unibe_ibu.config +++ b/conf/unibe_ibu.config @@ -1,25 +1,31 @@ params { config_profile_description = "University of Bern, Interfaculty Bioinformatics Unit cluster profile" - config_profile_contact = "irene.keller@dbmr.unibe.ch; info@bioinformatics.unibe.ch" + config_profile_contact = "alexander.nater@unibe.ch; info@bioinformatics.unibe.ch" config_profile_url = "https://www.bioinformatics.unibe.ch/" - max_memory = 500.GB - max_cpus = 128 - max_time = 240.h + schema_ignore_params = "project,clusterOptions" + project = null + clusterOptions = null +} + +validation { + ignoreParams = ["schema_ignore_params", "project", "clusterOptions"] } process { resourceLimits = [ memory: 500.GB, cpus: 128, - time: 240.h + time: 672.h ] - executor = "slurm" - maxRetries = 2 - beforeScript = 'mkdir -p ./tmp/ && export TMPDIR=./tmp/' + executor = 'slurm' + queue = 'pibu_el8' + maxRetries = 2 + scratch = '$SCRATCH' + clusterOptions = (params.project ? "-A ${params.project} " : '') + "${params.clusterOptions ?: ''}" } executor { - queueSize = 30 + queueSize = 50 } singularity { diff --git a/docs/icr_alma.md b/docs/icr_alma.md new file mode 100644 index 000000000..0474970f4 --- /dev/null +++ b/docs/icr_alma.md @@ -0,0 +1,41 @@ +# nf-core/configs: Institute of Cancer Research (Alma HPC) Configuration + +Deployment and testing of nf-core pipelines on the Alma cluster is on-going in collaboration between the RSE team and research groups. + +To run an nf-core pipeline on Alma, run the pipeline with `-profile icr_alma`. This will download and launch the [`icr_alma.config`](../conf/icr_alma.config) which has been pre-configured with a setup suitable for the Alma HPC cluster. + +Before running the pipeline you will need to load Nextflow using the environment module system. The preferred way is to create a condo or mamba environment (you only need to do this once) which you activate. + +```bash +## Create and activate mamba environment +mamba create --name mamba_nf -c bioconda nextflow nf-core +mamba activate mamba_nf +``` + +Singularity is installed on the compute nodes of Alma, but not the login nodes. There is no module for Singularity. + +All of the intermediate files required to run the pipeline will be stored in the `work/` directory. Using the `icr_alma` profile, the Work directory will be deleted upon the successful completion of the run. If a run fails, this automatic cleanup will not execute - you must manually delete the `work/` directory once debugged. + +> NB: Nextflow will need to submit the jobs via SLURM to the HPC cluster. This can be done from an interactive or a normal job. If in doubt contact Scientific Computing. + +Alma has a master-worker partition for the long running nextflow manager which spawns jobs to the compute node. A typical nextflow job will run a batch script thorugh sbatch which will run on the master-worker thread spawning the jobs. + +Command to run the nextflow job: + +```bash +sbatch my-nextflow.sh +``` + +Contents of my-nextflow.sh + +``` +#!/bin/bash +#SBATCH --job-name=nf-username +#SBATCH --output=slurm_out.txt +#SBATCH --partition=master-worker +#SBATCH --ntasks=1 +#SBATCH --time=1:00:00 +#SBATCH --mem-per-cpu=4000 + +nextflow run nf-core/sarek -profile icr_alma,test --outdir ./results +``` diff --git a/docs/seattlechildrens.md b/docs/seattlechildrens.md index 8e5c56a6b..8e6c3efa3 100644 --- a/docs/seattlechildrens.md +++ b/docs/seattlechildrens.md @@ -1,16 +1,20 @@ -# nf-core/configs: PROFILE Configuration +# nf-core/configs: Seattle Children's Research Institute Configuration -All nf-core pipelines have been successfully configured for use on the the Cybertron HPC at Seattle Children Research Institude (SCRI), Seattle, WA. +All nf-core pipelines have been successfully configured for use on the the Sasquatch HPC at Seattle Children Research Institude (SCRI), Seattle, WA. -To use, run the pipeline with `-profile PROFILENAME`. This will download and launch the pipeline using [`seattlechildrens.config`](../conf/seattlechildrens.config) which has been pre-configured with a setup suitable for the Cybertron cluster at SCRI. Using this profile, a container with all of the required software will be downloaded. +To use, run the pipeline with `-profile seattlechildrens`. This will download and launch the pipeline using [`seattlechildrens.config`](../conf/seattlechildrens.config) which has been pre-configured with a setup suitable for the Sasquatch cluster at SCRI. + +This profile assumes that you will use the `cpu-core-sponsored` partition. If you need to use `gpu-core-sponsored` for some steps, you can get in touch with Research Scientific Computing on Teams or email about how to modify the pipeline. + +We also maintain a [webpage about Nextflow](http://gonzo/hpcGuide/Nextflow.html) within the Seattle Children's intranet. # Project info -This config file is created for the use on the Cybertron HPC at Seattle Children Research Institude (SCRI), Seattle, WA. Using this config will pre-configure a set up suitable for the Cybertron HPC. The Singularity images will be downloaded to run on the cluster. The nextflow pipeline should be executed inside of the Cybertron system. +This config file is created for the use on the Sasquatch HPC at Seattle Children Research Institude (SCRI), Seattle, WA. Using this config will pre-configure a set up suitable for the Sasquatch HPC. The Singularity images will be downloaded to run on the cluster. The nextflow pipeline should be executed inside of the Sasquatch system. # Below are mandatory information SCRI -Before running the pipeline you will need to create a Nextflow environment on `mamba`. You can load _Singularity_ using the environment module system on **Cybertron**. +Before running the pipeline you will need to create a Nextflow environment on `mamba`. _Singularity_ is on the path by default and does not need to be loaded. ## Create a Nextflow `mamba` environment @@ -23,8 +27,8 @@ channels: - conda-forge dependencies: - python>=3.9 - - nextflow==23.10.0 - - nf-core==2.10 + - nextflow==24.10.4 + - nf-core==3.2.0 - graphviz ``` @@ -45,23 +49,25 @@ mamba config --set channel_priority flexible mamba env create -f nextflow.yaml ``` -4. Running in HPC (Cybertron) +4. Running in HPC (Sasquatch) Please look into [RSC-RP/nextflow_scri_config](https://github.com/RSC-RP/nextflow_scri_config) for details. ```bash # activate enviornment mamba activate nextflow -module load singularity -# to list all the projects with project codes you are authorized on HPC -project info +# To list all the accounts you are authorized on HPC. For example if you have an account cpu-mylab-sponsored, your association is "mylab". +sshare -o "Account,Partition%20" + +# Set your association as an environmental variable +export ASSOC="your_association_name" -# example to run nextflow pipeline (please replace with your own project code and module) -nextflow run -c 'conf/seattlechildrens.config' \ +# example to run nextflow pipeline (please replace with your own association, module, and temp directory) +nextflow run \ [nf-core/module_name] \ - -profile test,PBS_singularity \ - --project ["your_project_code"] \ + -profile seattlechildrens \ + -workDir /data/hps/assoc/private/mylab/user/mmouse/temp_rnaseq \ ``` You can find more information about computational resources [here](https:#child.seattlechildrens.org/research/center_support_services/research_informatics/research_scientific_computing/high_performance_computing_core/). You have to be an employee of SCRI to access the link. diff --git a/docs/unibe_ibu.md b/docs/unibe_ibu.md index 1adc40728..807ab0c04 100644 --- a/docs/unibe_ibu.md +++ b/docs/unibe_ibu.md @@ -2,4 +2,6 @@ Configuration file to run nf-core pipelines on the cluster of the [Interfaculty Bioinformatics Unit](https://www.bioinformatics.unibe.ch/) of the University of Bern. -To use, run the pipeline with `-profile unibe_ibu`. This will download and launch the profile.config which has been pre-configured with a setup suitable for the IBU cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. **This requires a local installation of singularity**. It is easiest to submit the pipeline from a compute node. Once the image is cached, you can also submit from the login node. +To use, run the pipeline with `-profile unibe_ibu` (one hyphen). This requires a local installation of singularity, so you have to run the pipeline from a compute node. + +For accounting, you can specify an IBU project id with the --project flag (two hyphens) when launching Nextflow. If no project is specified, it will default to the username. diff --git a/nfcore_custom.config b/nfcore_custom.config index a12864b59..86a05c40e 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -181,6 +181,9 @@ profiles { hypatia { includeConfig "${params.custom_config_base}/conf/hypatia.config" } + icr_alma{ + includeConfig "${params.custom_config_base}/conf/icr_alma.config" + } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } @@ -325,6 +328,9 @@ profiles { seadragon { includeConfig "${params.custom_config_base}/conf/seadragon.config" } + seattlechildrens { + includeConfig "${params.custom_config_base}/conf/seattlechildrens.config" + } seawulf { includeConfig "${params.custom_config_base}/conf/seawulf.config" } diff --git a/pipeline/drop.config b/pipeline/drop.config new file mode 100644 index 000000000..6ca97463a --- /dev/null +++ b/pipeline/drop.config @@ -0,0 +1,12 @@ +/* +* ------------------------------------------------- +* nfcore/drop custom profile Nextflow config file +* ------------------------------------------------- +* Config options for custom environments. +* Cluster-specific config options should be saved +* in the conf/pipeline/drop folder and imported +* under a profile name here. +*/ + +profiles { +}