Skip to content

Commit

Permalink
Merge pull request #769 from jfy133/batch-resourcelimit-update-pipeli…
Browse files Browse the repository at this point in the history 
…nelevel

Batch resourcelimit update pipeline-level institutional configs
  • Loading branch information
@jfy133
jfy133 authored Oct 7, 2024
2 parents 21a7b24 + 6055c45 commit 5bd8f99
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 110 deletions.
34 changes: 34 additions & 0 deletions conf/pipeline/eager/eva.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -618,3 +618,37 @@ profiles {
}
}
}

// Function to ensure that resource requirements don't go beyond
// a maximum limit
// FOR DSL1 PIPELINE ONLY!
def check_max(obj, type) {
if (type == 'memory') {
try {
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
return params.max_memory as nextflow.util.MemoryUnit
else
return obj
} catch (all) {
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'time') {
try {
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
return params.max_time as nextflow.util.Duration
else
return obj
} catch (all) {
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'cpus') {
try {
return Math.min( obj, params.max_cpus as int )
} catch (all) {
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
return obj
}
}
}
34 changes: 34 additions & 0 deletions conf/pipeline/eager/maestro.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,37 @@ profiles {
}
}
}

// Function to ensure that resource requirements don't go beyond
// a maximum limit
// FOR DSL1 PIPELINE ONLY!
def check_max(obj, type) {
if (type == 'memory') {
try {
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
return params.max_memory as nextflow.util.MemoryUnit
else
return obj
} catch (all) {
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'time') {
try {
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
return params.max_time as nextflow.util.Duration
else
return obj
} catch (all) {
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'cpus') {
try {
return Math.min( obj, params.max_cpus as int )
} catch (all) {
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
return obj
}
}
}
98 changes: 49 additions & 49 deletions conf/pipeline/mag/eva.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,30 @@ process {
}

withLabel: process_single {
cpus = { check_max(1, 'cpus') }
memory = { check_max(6.GB * task.attempt, 'memory') }
cpus = 1
memory = { 6.GB * task.attempt }
time = 365.d
}
withLabel: process_low {
cpus = { check_max(2 * task.attempt, 'cpus') }
memory = { check_max(12.GB * task.attempt, 'memory') }
cpus = { 2 * task.attempt }
memory = { 12.GB * task.attempt }
time = 365.d
}
withLabel: process_medium {
cpus = { check_max(6 * task.attempt, 'cpus') }
memory = { check_max(36.GB * task.attempt, 'memory') }
cpus = { 6 * task.attempt }
memory = { 36.GB * task.attempt }
time = 365.d
}
withLabel: process_high {
cpus = { check_max(12 * task.attempt, 'cpus') }
memory = { check_max(72.GB * task.attempt, 'memory') }
cpus = { 12 * task.attempt }
memory = { 72.GB * task.attempt }
time = 365.d
}
withLabel: process_long {
time = 365.d
}
withLabel: process_high_memory {
memory = { check_max(200.GB * task.attempt, 'memory') }
memory = { 200.GB * task.attempt }
time = 365.d
}
withLabel: error_ignore {
Expand All @@ -59,117 +59,117 @@ process {
}

withName: BOWTIE2_HOST_REMOVAL_BUILD {
cpus = { check_max(10 * task.attempt, 'cpus') }
memory = { check_max(20.GB * task.attempt, 'memory') }
cpus = { 10 * task.attempt }
memory = { 20.GB * task.attempt }
time = 365.d
}
withName: BOWTIE2_HOST_REMOVAL_ALIGN {
cpus = { check_max(10 * task.attempt, 'cpus') }
memory = { check_max(10.GB * task.attempt, 'memory') }
cpus = { 10 * task.attempt }
memory = { 10.GB * task.attempt }
time = 365.d
}
withName: BOWTIE2_PHIX_REMOVAL_ALIGN {
cpus = { check_max(4 * task.attempt, 'cpus') }
memory = { check_max(8.GB * task.attempt, 'memory') }
cpus = { 4 * task.attempt }
memory = { 8.GB * task.attempt }
time = 365.d
}
withName: PORECHOP {
cpus = { check_max(4 * task.attempt, 'cpus') }
memory = { check_max(30.GB * task.attempt, 'memory') }
cpus = { 4 * task.attempt }
memory = { 30.GB * task.attempt }
time = 365.d
}
withName: NANOLYSE {
cpus = { check_max(2 * task.attempt, 'cpus') }
memory = { check_max(10.GB * task.attempt, 'memory') }
cpus = { 2 * task.attempt }
memory = { 10.GB * task.attempt }
time = 365.d
}
//filtlong: exponential increase of memory and time with attempts
withName: FILTLONG {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(64.GB * (2 ** (task.attempt - 1)), 'memory') }
cpus = { 8 * task.attempt }
memory = { 64.GB * (2 ** (task.attempt - 1)) }
time = 365.d
}
withName: CENTRIFUGE {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(40.GB * task.attempt, 'memory') }
cpus = { 8 * task.attempt }
memory = { 40.GB * task.attempt }
time = 365.d
}
withName: KRAKEN2 {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(40.GB * task.attempt, 'memory') }
cpus = { 8 * task.attempt }
memory = { 40.GB * task.attempt }
time = 365.d
}
withName: KRONA {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(20.GB * task.attempt, 'memory') }
cpus = { 8 * task.attempt }
memory = { 20.GB * task.attempt }
time = 365.d
}
withName: CAT_DB_GENERATE {
memory = { check_max(200.GB * task.attempt, 'memory') }
memory = { 200.GB * task.attempt }
time = 365.d
}
withName: CAT {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(40.GB * task.attempt, 'memory') }
cpus = { 8 * task.attempt }
memory = { 40.GB * task.attempt }
time = 365.d
}
withName: GTDBTK_CLASSIFYWF {
cpus = { check_max(10 * task.attempt, 'cpus') }
memory = { check_max(128.GB * task.attempt, 'memory') }
cpus = { 10 * task.attempt }
memory = { 128.GB * task.attempt }
time = 365.d
}
//MEGAHIT returns exit code 250 when running out of memory
withName: MEGAHIT {
cpus = { check_megahit_cpus(8, task.attempt) }
memory = { check_max(40.GB * task.attempt, 'memory') }
memory = { 40.GB * task.attempt }
time = 365.d
errorStrategy = { task.exitStatus in ((130..145) + 104 + 250) ? 'retry' : 'finish' }
}
//SPAdes returns error(1) if it runs out of memory (and for other reasons as well...)!
//exponential increase of memory and time with attempts, keep number of threads to enable reproducibility
withName: SPADES {
cpus = { check_spades_cpus(10, task.attempt) }
memory = { check_max(64.GB * (2 ** (task.attempt - 1)), 'memory') }
memory = { 64.GB * (2 ** (task.attempt - 1)) }
time = 365.d
errorStrategy = { task.exitStatus in [143, 137, 21, 1] ? 'retry' : 'finish' }
maxRetries = 5
}
withName: SPADESHYBRID {
cpus = { check_spadeshybrid_cpus(10, task.attempt) }
memory = { check_max(64.GB * (2 ** (task.attempt - 1)), 'memory') }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
memory = { 64.GB * (2 ** (task.attempt - 1)) }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
errorStrategy = { task.exitStatus in [143, 137, 21, 1] ? 'retry' : 'finish' }
maxRetries = 5
}
//returns exit code 247 when running out of memory
withName: BOWTIE2_ASSEMBLY_ALIGN {
cpus = { check_max(2 * task.attempt, 'cpus') }
memory = { check_max(8.GB * task.attempt, 'memory') }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
cpus = { 2 * task.attempt }
memory = { 8.GB * task.attempt }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139, 247] ? 'retry' : 'finish' }
}
withName: METABAT2_METABAT2 {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(20.GB * task.attempt, 'memory') }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
cpus = { 8 * task.attempt }
memory = { 20.GB * task.attempt }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
}
withName: MAG_DEPTHS {
memory = { check_max(16.GB * task.attempt, 'memory') }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
memory = { 16.GB * task.attempt }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
}
withName: BUSCO {
cpus = { check_max(8 * task.attempt, 'cpus') }
memory = { check_max(20.GB * task.attempt, 'memory') }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
cpus = { 8 * task.attempt }
memory = { 20.GB * task.attempt }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
}

withName: MAXBIN2 {
errorStrategy = { task.exitStatus in [1, 255] ? 'ignore' : 'retry' }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
}

withName: DASTOOL_DASTOOL {
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : task.exitStatus == 1 ? 'ignore' : 'finish' }
time = { check_max(24.h * (2 ** (task.attempt - 1)), 'time') }
time = { 24.h * (2 ** (task.attempt - 1)), 'time') }
}
}
Loading

0 comments on commit 5bd8f99

Please sign in to comment.