configs/conf/seadragon.config at 62b5da4d9e2f6ceae166d2112b2957ae62f82735 · ICR-RSE-Group/configs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
//Profile config names for nf-core/configs
params {
    config_profile_description = 'University of Texas at MD Anderson HPC cluster profile provided by nf-core/configs'
    config_profile_contact = 'Jiawei Gu'
    config_profile_contact_github = '@jiawku'
    config_profile_contact_email = 'jiawei.gu@mdanderson.org'
    config_profile_url = 'https://hpcweb.mdanderson.edu/'
}

env {
    SINGULARITY_CACHEDIR="/home/$USER/.singularity/cache"
    APPTAINER_TMPDIR="/home/$USER/.apptainer/tmp"
    APPTAINER_CACHEDIR="/home/$USER/.apptainer/cache"
}

singularity {
    enabled = true
    envWhitelist='APPTAINERENV_NXF_TASK_WORKDIR,APPTAINERENV_NXF_DEBUG,APPTAINERENV_LD_LIBRARY_PATH,SINGULARITY_BINDPATH,LD_LIBRARY_PATH,TMPDIR,SINGULARITY_TMPDIR'
    autoMounts = true
    runOptions = '-B ${TMPDIR:-/tmp}'
    cacheDir = "/home/$USER/.singularity/cache"
}

def membership = "groups".execute().text

def select_queue = { memory, cpu, walltime ->
    if (memory <= 950.GB && cpu <= 80) {
        if (walltime <= 3.h) return 'short'
        if (walltime <= 24.h) return 'medium'
        if (walltime <= 240.h) return 'long'
        if (walltime <= 504.h) return 'vlong'
    }

    // High memory queues
    if (memory <= 3900.GB && cpu <= 80 && walltime <= 504.h) {
        if (walltime <= 240.h) return 'evhighmem'
    }

    throw new IllegalArgumentException("No matching queue for memory=${memory}, cpu=${cpu}, time=${time}")

}

// Submit up to 100 concurrent jobs
// pollInterval and queueStatInterval of every 5 minutes
// submitRateLimit of 20 per minute
executor {
    name = 'lsf'
    queueSize = 100
    perJobMemLimit = true
    queueStatInterval = '2 min'
    submitRateLimit = '20 min'
    jobName = { "${task.process.split(':').last()}" }
}


process {
    resourceLimits = [
        memory: 3900.GB, // Max memory based on vhighmem node
        cpus: 80,       // Max CPUs based on E80 node
        time: 504.h     // Max time for long queues
    ]

    executor = 'lsf' // Use LSF executor

    cpus   = { 2      * task.attempt }
    memory = { 12.GB   * task.attempt }
    time   = { 3.h    * task.attempt }

    maxRetries     = 3
    afterScript   = 'sleep 10' // Prevent abrupt re-submissions after retries
    queue = { select_queue(task.memory, task.cpus, task.time) } // Use the updated select_queue function

    withLabel:process_gpu {
        cpus   = { 40 }           // Use Gdragon nodes
        memory = { 168.GB }       // Max memory for GPU nodes
        queue  = 'gpu,gpu-medium' // Specific GPU queues
    }
}

params {
    max_memory = 3900.GB  // Maximum memory based on evhighmem nodes
    max_cpus = 80        // Maximum CPUs based on E80 nodes
    max_time = 504.h     // Maximum runtime for evlong queues
    igenomes_base = '/rsrch3/scratch/reflib/REFLIB_data/AWS-iGenomes'
}

cleanup = true