Skip to content

Commit 7a9d5f6

Browse files
HenrikBengtssonHenrikBengtsson
committed
batchtools_sge(): help + tweak template script
1 parent 40e7411 commit 7a9d5f6

File tree

9 files changed

+231
-192
lines changed

9 files changed

+231
-192
lines changed

‎DESCRIPTION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Package: future.batchtools
2-
Version: 0.12.2-9919
2+
Version: 0.12.2-9920
33
Depends:
44
R (>= 3.2.0),
55
parallelly,

‎R/batchtools_sge.R‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#' @export
2+
BatchtoolsSGEFutureBackend <- function(...) {
3+
core <- BatchtoolsTemplateFutureBackend(..., type = "sge")
4+
core[["futureClasses"]] <- c("BatchtoolsSGEFuture", core[["futureClasses"]])
5+
core <- structure(core, class = c("BatchtoolsSGEFutureBackend", class(core)))
6+
core
7+
}
8+
9+
#' A batchtools SGE backend resolves futures in parallel via a SGE job scheduler
10+
#'
11+
#' @inheritParams BatchtoolsFutureBackend
12+
#' @inheritParams BatchtoolsTemplateFutureBackend
13+
#'
14+
#' @param template (optional) Name of job-script template to be searched
15+
#' for by [batchtools::findTemplateFile()]. If not found, it defaults to
16+
#' the `templates/sge.tmpl` part of this package (see below).
17+
#'
18+
#' @param \ldots Not used.
19+
#'
20+
#' @details
21+
#' Batchtools sge futures use \pkg{batchtools} cluster functions
22+
#' created by [batchtools::makeClusterFunctionsSGE()], which requires
23+
#' that SGE commands `qsub`, `qstat`, and `qdel` are installed on
24+
#' the current machine.
25+
#'
26+
#' The default template script `templates/sge.tmpl` can be found in:
27+
#'
28+
#' ```r
29+
#' system.file("templates", "sge.tmpl", package = "future.batchtools")
30+
#' ```
31+
#'
32+
#' and comprise:
33+
#'
34+
#' `r paste(c("\x60\x60\x60bash", readLines("inst/templates/sge.tmpl"), "\x60\x60\x60"), collapse = "\n")`
35+
#'
36+
#' @examplesIf interactive()
37+
#' # Limit runtime to 3 minutes and memory to 200 MiB per future
38+
#' plan(batchtools_sge, resources = list(h_rt = "00:03:00", mem_free = "200M"))
39+
#'
40+
#' message("Main process ID: ", Sys.getpid())
41+
#'
42+
#' f <- future(Sys.getpid())
43+
#' pid <- value(f)
44+
#' message("Worker process ID: ", pid)
45+
#'
46+
#' @export
47+
batchtools_sge <- function(..., template = "sge") {
48+
stop("INTERNAL ERROR: The future.batchtools::batchtools_sge() must never be called directly")
49+
}
50+
class(batchtools_sge) <- c(
51+
"batchtools_sge", "batchtools_template",
52+
"batchtools_multiprocess", "batchtools",
53+
"multiprocess", "future", "function"
54+
)
55+
attr(batchtools_sge, "tweakable") <- c(
56+
"workers",
57+
"finalize",
58+
## Arguments to batchtools::makeClusterFunctionsSGE()
59+
"nodename", "scheduler.latency", "fs.latency"
60+
)
61+
attr(batchtools_sge, "init") <- TRUE
62+
attr(batchtools_sge, "factory") <- BatchtoolsSGEFutureBackend

‎R/batchtools_slurm.R‎

Lines changed: 1 addition & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,91 +1,3 @@
1-
#' Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc.
2-
#'
3-
#' Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc. are
4-
#' asynchronous multiprocess futures that will be evaluated on a compute
5-
#' cluster via a job scheduler.
6-
#'
7-
#' @inheritParams BatchtoolsFutureBackend
8-
#'
9-
#' @param template (optional) A batchtools template file or a template string
10-
#' (in \pkg{brew} format). If not specified, it is left to the
11-
#' \pkg{batchtools} package to locate such file using its search rules.
12-
#'
13-
#' @param type (character string) Type of job scheduler.
14-
#'
15-
#' @param \ldots Additional arguments passed to [BatchtoolsFutureBackend()].
16-
#'
17-
#' @return An object of class `BatchtoolsFutureBackend`.
18-
#'
19-
#' @details
20-
#' These type of batchtools futures rely on batchtools backends set
21-
#' up using the following \pkg{batchtools} functions:
22-
#'
23-
#' * [batchtools::makeClusterFunctionsLSF()] for
24-
#' [Load Sharing Facility (LSF)](https://en.wikipedia.org/wiki/Platform_LSF)
25-
#' * [batchtools::makeClusterFunctionsOpenLava()] for
26-
#' [OpenLava](https://en.wikipedia.org/wiki/OpenLava)
27-
#' * [batchtools::makeClusterFunctionsSGE()] for
28-
#' [Sun/Oracle Grid Engine (SGE)](https://en.wikipedia.org/wiki/Oracle_Grid_Engine)
29-
#' * [batchtools::makeClusterFunctionsSlurm()] for
30-
#' [Slurm](https://en.wikipedia.org/wiki/Slurm_Workload_Manager)
31-
#' * [batchtools::makeClusterFunctionsTORQUE()] for
32-
#' [TORQUE](https://en.wikipedia.org/wiki/TORQUE) / PBS
33-
#'
34-
#' @aliases BatchtoolsLsfFutureBackend BatchtoolsOpenLavaFutureBackend BatchtoolsSGEFutureBackend BatchtoolsSlurmFutureBackend BatchtoolsTorqueFutureBackend
35-
#' @aliases batchtools_lsf batchtools_openlava batchtools_sge batchtools_torque
36-
#'
37-
#' @importFrom batchtools makeClusterFunctionsLSF
38-
#' @importFrom batchtools makeClusterFunctionsOpenLava
39-
#' @importFrom batchtools makeClusterFunctionsSGE
40-
#' @importFrom batchtools makeClusterFunctionsSlurm
41-
#' @importFrom batchtools makeClusterFunctionsTORQUE
42-
#' @export
43-
BatchtoolsTemplateFutureBackend <- function(..., template = NULL, type = c("lsf", "openlava", "sge", "slurm", "torque")) {
44-
assert_no_positional_args_but_first()
45-
type <- match.arg(type)
46-
47-
48-
dotdotdot <- list(...)
49-
50-
make_cfs <- switch(type,
51-
lsf = makeClusterFunctionsLSF,
52-
openlava = makeClusterFunctionsOpenLava,
53-
sge = makeClusterFunctionsSGE,
54-
slurm = makeClusterFunctionsSlurm,
55-
torque = makeClusterFunctionsTORQUE
56-
)
57-
58-
make_cfs_formals <- formals(make_cfs)
59-
60-
## Get the default template?
61-
if (is.null(template)) {
62-
template <- make_cfs_formals$template
63-
}
64-
65-
stop_if_not(is.character(template), length(template) == 1L,
66-
!is.na(template), nzchar(template))
67-
68-
template <- find_template_file(template)
69-
70-
keep <- which(names(dotdotdot) %in% names(make_cfs_formals))
71-
args <- c(list(template = template), dotdotdot[keep])
72-
cluster.functions <- do.call(make_cfs, args = args)
73-
attr(cluster.functions, "template") <- template
74-
75-
## Drop used '...' arguments
76-
if (length(keep) > 0) dotdotdot <- dotdotdot[-keep]
77-
78-
args <- dotdotdot
79-
args[["cluster.functions"]] <- cluster.functions
80-
81-
core <- do.call(BatchtoolsMultiprocessFutureBackend, args = args)
82-
83-
core[["futureClasses"]] <- c("BatchtoolsTemplateFuture", "BatchtoolsMultiprocessFuture", core[["futureClasses"]])
84-
core <- structure(core, class = c("BatchtoolsTemplateFutureBackend", class(core)))
85-
core
86-
}
87-
88-
891
#' @export
902
BatchtoolsSlurmFutureBackend <- function(...) {
913
core <- BatchtoolsTemplateFutureBackend(..., type = "slurm")
@@ -133,7 +45,7 @@ BatchtoolsSlurmFutureBackend <- function(...) {
13345
#' message("Worker process ID: ", pid)
13446
#'
13547
#' @export
136-
batchtools_slurm <- function(...) {
48+
batchtools_slurm <- function(..., template = "slurm") {
13749
stop("INTERNAL ERROR: The future.batchtools::batchtools_slurm() must never be called directly")
13850
}
13951
class(batchtools_slurm) <- c(

‎R/batchtools_template.R‎

Lines changed: 1 addition & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
#' [TORQUE](https://en.wikipedia.org/wiki/TORQUE) / PBS
3333
#'
3434
#' @aliases BatchtoolsLsfFutureBackend BatchtoolsOpenLavaFutureBackend BatchtoolsSGEFutureBackend BatchtoolsSlurmFutureBackend BatchtoolsTorqueFutureBackend
35-
#' @aliases batchtools_lsf batchtools_openlava batchtools_sge batchtools_torque
35+
#' @aliases batchtools_lsf batchtools_openlava batchtools_torque
3636
#'
3737
#' @importFrom batchtools makeClusterFunctionsLSF
3838
#' @importFrom batchtools makeClusterFunctionsOpenLava
@@ -141,62 +141,6 @@ attr(batchtools_openlava, "factory") <- BatchtoolsOpenLavaFutureBackend
141141

142142

143143

144-
#' @export
145-
BatchtoolsSGEFutureBackend <- function(...) {
146-
core <- BatchtoolsTemplateFutureBackend(..., type = "sge")
147-
core[["futureClasses"]] <- c("BatchtoolsSGEFuture", core[["futureClasses"]])
148-
core <- structure(core, class = c("BatchtoolsSGEFutureBackend", class(core)))
149-
core
150-
}
151-
152-
#' @export
153-
batchtools_sge <- function(...) {
154-
stop("INTERNAL ERROR: The future.batchtools::batchtools_sge() must never be called directly")
155-
}
156-
class(batchtools_sge) <- c(
157-
"batchtools_sge", "batchtools_template",
158-
"batchtools_multiprocess", "batchtools",
159-
"multiprocess", "future", "function"
160-
)
161-
attr(batchtools_sge, "tweakable") <- c(
162-
"workers",
163-
"finalize",
164-
## Arguments to batchtools::makeClusterFunctionsSGE()
165-
"nodename", "scheduler.latency", "fs.latency"
166-
)
167-
attr(batchtools_sge, "init") <- TRUE
168-
attr(batchtools_sge, "factory") <- BatchtoolsSGEFutureBackend
169-
170-
171-
172-
#' @export
173-
BatchtoolsSlurmFutureBackend <- function(...) {
174-
core <- BatchtoolsTemplateFutureBackend(..., type = "slurm")
175-
core[["futureClasses"]] <- c("BatchtoolsSlurmFuture", core[["futureClasses"]])
176-
core <- structure(core, class = c("BatchtoolsSlurmFutureBackend", class(core)))
177-
core
178-
}
179-
180-
#' @export
181-
batchtools_slurm <- function(...) {
182-
stop("INTERNAL ERROR: The future.batchtools::batchtools_slurm() must never be called directly")
183-
}
184-
class(batchtools_slurm) <- c(
185-
"batchtools_slurm", "batchtools_template",
186-
"batchtools_multiprocess", "batchtools",
187-
"multiprocess", "future", "function"
188-
)
189-
attr(batchtools_slurm, "tweakable") <- c(
190-
"workers",
191-
"finalize",
192-
## Arguments to batchtools::makeClusterFunctionsSlurm()
193-
"array.jobs", "nodename", "scheduler.latency", "fs.latency"
194-
)
195-
attr(batchtools_slurm, "init") <- TRUE
196-
attr(batchtools_slurm, "factory") <- BatchtoolsSlurmFutureBackend
197-
198-
199-
200144
#' @export
201145
BatchtoolsTorqueFutureBackend <- function(...) {
202146
core <- BatchtoolsTemplateFutureBackend(..., type = "torque")

‎inst/templates/sge.tmpl‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88
#$ -S /bin/bash # Run this as a bash script (required)
99

1010
## Job name:
11-
#$ -N <%= if (exists("job.name", mode = "character")) job.name else job.hash %>
12-
13-
## Merge standard error and output:
14-
#$ -j y
11+
#$ -N <%= job.name %>
1512

1613
## Direct streams to logfile:
1714
#$ -o <%= log.file %>
1815

16+
## Merge standard error and output:
17+
#$ -j y
18+
1919
## Tell the queue system to use the current directory
2020
## as the working directory
2121
#$ -cwd
@@ -32,29 +32,32 @@
3232
cat(sprintf("#$ %s\n", resources[["asis"]]))
3333
resources <- resources[names(resources) != "asis"]
3434
}
35-
35+
3636
## Remaining resources are assumed to be of type '-l', e.g.
3737
## plan(batchtools_sge, resources = list(mem_free = "1G", h_rt="00:20:00"))
3838
opts <- unlist(resources, use.names = TRUE)
3939
opts <- sprintf("-l %s=%s", names(opts), opts)
4040
cat(sprintf("#$ %s\n", opts))
4141
} %>
4242

43-
# For troubleshooting, in case there are errors
43+
echo "Batchtools job name: '<%= job.name %>'"
44+
45+
echo "Session information:"
4446
date
4547
hostname
4648
which Rscript
4749
Rscript --version
4850
Rscript -e ".libPaths()"
4951

52+
## Launch R and evaluate the batchtools R job
5053
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
5154
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
5255
res=$?
5356
echo " - exit code: ${res}"
5457
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
5558

56-
## For troubleshooting if there are errors
57-
qstat -j $JOB_ID
59+
## End-of-job summary
60+
qstat -j "${JOB_ID}"
5861

5962
## Relay the exit code from Rscript
60-
exit "$res"
63+
exit "${res}"

‎inst/templates/slurm.tmpl‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77

88
<% Slurm defaults
99
defaults <- list(
10-
nodes = 1, # single-host processing
11-
time = "00:05:00", # 5-min runtime
12-
mem = "100M" # 100 MiB memory
10+
nodes = 1, # single-host processing
11+
time = "00:05:00", # 5-min runtime
12+
mem = "100M" # 100 MiB memory
1313
)
1414
resources <- c(resources, defaults[setdiff(names(defaults), names(resources))])
1515
opts <- unlist(resources, use.names = TRUE)
@@ -21,5 +21,24 @@ opts <- paste(opts, collapse = " ") %>
2121
#SBATCH --output=<%= log.file %>
2222
#SBATCH <%= opts %>
2323

24+
echo "Batchtools job name: '<%= job.name %>'"
25+
26+
echo "Session information:"
27+
date
28+
hostname
29+
which Rscript
30+
Rscript --version
31+
Rscript -e ".libPaths()"
32+
2433
## Launch R and evaluate the batchtools R job
34+
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
2535
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
36+
res=$?
37+
echo " - exit code: ${res}"
38+
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
39+
40+
## End-of-job summary
41+
sstat --format="JobID,AveCPU,MaxRSS,MaxPages,MaxDiskRead,MaxDiskWrite" --allsteps --jobs="${SLURM_JOB_ID}"
42+
43+
## Relay the exit code from Rscript
44+
exit "${res}"

‎man/BatchtoolsTemplateFutureBackend.Rd‎

Lines changed: 1 addition & 29 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)