Skip to content

Commit 53d71ff

Browse files
HenrikBengtssonHenrikBengtsson
committed
Add support for controlling the 'Rscript' call in the built-in job script templates [#99]
1 parent 621ca01 commit 53d71ff

17 files changed

+366
-174
lines changed

‎DESCRIPTION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Package: future.batchtools
2-
Version: 0.20.0-9019
2+
Version: 0.20.0-9020
33
Depends:
44
R (>= 3.2.0),
55
parallelly,

‎NEWS.md‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,15 @@
66

77
## New Features
88

9+
* Add support for controlling the 'Rscript' call in the built-in job
10+
script templates. This can be done via fields 'rscript' and
11+
'rscript_args' of the 'resources' argument.
12+
13+
* The built-in job script templates assert that the `Rscript`
14+
launcher is found, and if not, they give an informative error
15+
message suggesting to declare environment modules, via the
16+
`resources` argument, that should be loaded by the job script.
17+
918
* Add `makeClusterFunctionsSlurm2()`, which patches
1019
`batchtools::makeClusterFunctionsSlurm()`. Firstly, it patches the
1120
`listJobsQueued()` cluster function such that it falls back to
@@ -17,11 +26,6 @@
1726
from stdout, which prevents auxillary INFO messages from `sbatch`
1827
to corrupt the output to be parsed.
1928

20-
* The built-in job script templates assert that `Rscript` is found,
21-
and if not, they give an informative error message suggesting to
22-
declare environment modules, via the `resources` argument, that
23-
should be loaded by the job script.
24-
2529
## Bug Fixes
2630

2731
`batchtools_slurm()` would produce "Future of class

‎R/BatchtoolsFutureBackend-class.R‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@
3838
#' vectors of shell code to be injected to the job script as-is.
3939
#' * `resources[["details"]]`, if TRUE, results in the job script outputting
4040
#' job details and job summaries at the beginning and at the end.
41+
#' * `resources[["rscript"]]` is an optional character vector specifying
42+
#' how the 'Rscript' is launched. The `resources[["rscript_args"]]` field
43+
#' is an optional character vector specifying the 'Rscript' command-line
44+
#' arguments.
4145
#' * All remaining `resources` named elements are injected as named resource
4246
#' specification for the scheduler.
4347
#'

‎inst/templates/bash.tmpl‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,6 @@ trap 'echo "ERROR: future.batchtools job script failed on line $LINENO" >&2; exi
1515
exec > <%= log.file %> 2>&1
1616

1717
<%
18-
## Maximum runtime?
19-
runtime <- resources[["timeout"]]
20-
resources[["timeout"]] <- NULL
21-
timeout <- if (is.null(runtime)) "" else sprintf("timeout %s", runtime)
22-
2318
## Shell "startup" code to evaluate
2419
startup <- resources[["startup"]]
2520
resources[["startup"]] <- NULL
@@ -31,6 +26,26 @@ exec > <%= log.file %> 2>&1
3126
## Environment modules specifications
3227
modules <- resources[["modules"]]
3328
resources[["modules"]] <- NULL
29+
30+
## Custom "Rscript" command and Rscript arguments
31+
rscript <- resources[["rscript"]]
32+
if (is.null(rscript)) {
33+
rscript <- "Rscript"
34+
} else {
35+
if (length(rscript) == 0 || !nzchar(rscript)[1]) stop("Argument 'resources' specifies an empty 'rscript' field")
36+
rscript[rscript == "*"] <- "Rscript"
37+
}
38+
resources[["rscript"]] <- NULL
39+
40+
## Maximum runtime?
41+
timeout <- resources[["timeout"]]
42+
resources[["timeout"]] <- NULL
43+
if (length(timeout) > 0) {
44+
rscript <- c("timeout", timeout, rscript)
45+
}
46+
rscript_args <- resources[["rscript_args"]]
47+
resources[["rscript_args"]] <- NULL
48+
rscript_call <- paste(c(rscript, rscript_args), collapse = " ")
3449
%>
3550

3651
<% if (length(startup) > 0) {
@@ -49,20 +64,26 @@ exec > <%= log.file %> 2>&1
4964
echo "Session information:"
5065
echo "- timestamp: $(date +"%Y-%m-%d %H:%M:%S%z")"
5166
echo "- hostname: $(hostname)"
52-
echo "- Rscript path: $(which Rscript)"
53-
echo "- Rscript version: $(Rscript --version)"
54-
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
67+
echo "- Rscript: <%= paste(rscript, collapse = " ") %>"
68+
echo "- Rscript args: <%= paste(rscript_args, collapse = " ") %>"
69+
echo "- Rscript call: <%= rscript_call %>"
70+
if ! command -v <%= rscript[1] %> &> /dev/null; then
71+
>&2 echo "ERROR: Argument 'resources' specifies a non-existing 'Rscript' launch command: <%= rscript[1] %>. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_slurm, resources = list(modules = c(\"r\")))'. The search PATH for '%<= rscript[1] %>' was ${PATH}"
72+
exit 1
73+
fi
74+
echo "- Rscript version: $(<%= paste(rscript, collapse = " ") %> --version)"
75+
echo "- Rscript library paths: $(<%= rscript_call %> -e "cat(shQuote(.libPaths()), sep = ' ')")"
5576
echo
5677

5778
# Launch R and evaluate the batchtools R job
58-
echo "Rscript -e 'batchtools::doJobCollection()' ..."
79+
echo "Calling 'batchtools::doJobCollection()' ..."
5980
echo "- job name: '<%= job.name %>'"
6081
echo "- job log file: '<%= log.file %>'"
6182
echo "- job uri: '<%= uri %>'"
62-
<%= timeout %> Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
83+
<%= rscript_call %> -e 'batchtools::doJobCollection("<%= uri %>")'
6384
res=$?
6485
echo " - exit code: ${res}"
65-
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
86+
echo "Calling 'batchtools::doJobCollection()' ... done"
6687
echo
6788

6889
<% if (length(shutdown) > 0) {

‎inst/templates/lsf.tmpl‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
######################################################################
3-
# A batchtools launch script template for LSF
3+
# A batchtools launch script template for LSF and OpenLava
44
#
55
# Author: Henrik Bengtsson
66
######################################################################
@@ -29,14 +29,27 @@
2929
modules <- resources[["modules"]]
3030
resources[["modules"]] <- NULL
3131

32+
## Custom "Rscript" command and Rscript arguments
33+
rscript <- resources[["rscript"]]
34+
if (is.null(rscript)) {
35+
rscript <- "Rscript"
36+
} else {
37+
if (length(rscript) == 0 || !nzchar(rscript)[1]) stop("Argument 'resources' specifies an empty 'rscript' field")
38+
rscript[rscript == "*"] <- "Rscript"
39+
}
40+
resources[["rscript"]] <- NULL
41+
rscript_args <- resources[["rscript_args"]]
42+
resources[["rscript_args"]] <- NULL
43+
rscript_call <- paste(c(rscript, rscript_args), collapse = " ")
44+
3245
## As-is resource specifications
3346
job_declarations <- resources[["asis"]]
3447
resources[["asis"]] <- NULL
3548

36-
## Remaining resources are assumed to be of type '-<key>=<value>'
49+
## Remaining resources are assumed to be of type '<key>=<value>'
3750
opts <- unlist(resources, use.names = TRUE)
38-
opts <- sprintf("-%s=%s", names(opts), opts)
39-
job_declarations <- sprintf("#BSUB %s", c(job_declarations, opts))
51+
opts <- sprintf("%s=%s", names(opts), opts)
52+
job_declarations <- sprintf("#BSUB %s", c(job_declarations, sprintf("-%s", opts)))
4053
writeLines(job_declarations)
4154
%>
4255

@@ -78,27 +91,25 @@ fi
7891
echo "Session information:"
7992
echo "- timestamp: $(date +"%Y-%m-%d %H:%M:%S%z")"
8093
echo "- hostname: $(hostname)"
81-
82-
if command -v Rscript &> /dev/null; then
83-
>&2 echo "ERROR: 'Rscript' could not be found. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_lsf, resources = list(modules = c(\"r\")))'. The search PATH for 'Rscript' was ${PATH}"
94+
echo "- Rscript call: <%= rscript_call %>"
95+
if ! command -v <%= rscript[1] %> &> /dev/null; then
96+
>&2 echo "ERROR: Argument 'resources' specifies a non-existing 'Rscript' launch command: <%= rscript[1] %>. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_slurm, resources = list(modules = c(\"r\")))'. The search PATH for '%<= rscript[1] %>' was ${PATH}"
8497
exit 1
8598
fi
86-
87-
echo "- Rscript path: $(which Rscript)"
88-
echo "- Rscript version: $(Rscript --version)"
89-
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
99+
echo "- Rscript version: $(<%= paste(rscript, collapse = " ") %> --version)"
100+
echo "- Rscript library paths: $(<%= rscript_call %> -e "cat(shQuote(.libPaths()), sep = ' ')")"
90101
echo
91102

92103

93104
## Launch R and evaluate the batchtools R job
94-
echo "Rscript -e 'batchtools::doJobCollection()' ..."
105+
echo "Calling 'batchtools::doJobCollection()' ..."
95106
echo "- job name: '<%= job.name %>'"
96107
echo "- job log file: '<%= log.file %>'"
97108
echo "- job uri: '<%= uri %>'"
98-
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
109+
<%= rscript_call %> -e 'batchtools::doJobCollection("<%= uri %>")'
99110
res=$?
100111
echo " - exit code: ${res}"
101-
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
112+
echo "Calling 'batchtools::doJobCollection()' ... done"
102113
echo
103114

104115
<% if (details) { %>

‎inst/templates/openlava.tmpl‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
######################################################################
3-
# A batchtools launch script template for OpenLava
3+
# A batchtools launch script template for LSF and OpenLava
44
#
55
# Author: Henrik Bengtsson
66
######################################################################
@@ -29,14 +29,27 @@
2929
modules <- resources[["modules"]]
3030
resources[["modules"]] <- NULL
3131

32+
## Custom "Rscript" command and Rscript arguments
33+
rscript <- resources[["rscript"]]
34+
if (is.null(rscript)) {
35+
rscript <- "Rscript"
36+
} else {
37+
if (length(rscript) == 0 || !nzchar(rscript)[1]) stop("Argument 'resources' specifies an empty 'rscript' field")
38+
rscript[rscript == "*"] <- "Rscript"
39+
}
40+
resources[["rscript"]] <- NULL
41+
rscript_args <- resources[["rscript_args"]]
42+
resources[["rscript_args"]] <- NULL
43+
rscript_call <- paste(c(rscript, rscript_args), collapse = " ")
44+
3245
## As-is resource specifications
3346
job_declarations <- resources[["asis"]]
3447
resources[["asis"]] <- NULL
3548

36-
## Remaining resources are assumed to be of type '-<key>=<value>'
49+
## Remaining resources are assumed to be of type '<key>=<value>'
3750
opts <- unlist(resources, use.names = TRUE)
38-
opts <- sprintf("-%s=%s", names(opts), opts)
39-
job_declarations <- sprintf("#BSUB %s", c(job_declarations, opts))
51+
opts <- sprintf("%s=%s", names(opts), opts)
52+
job_declarations <- sprintf("#BSUB %s", c(job_declarations, sprintf("-%s", opts)))
4053
writeLines(job_declarations)
4154
%>
4255

@@ -78,27 +91,25 @@ fi
7891
echo "Session information:"
7992
echo "- timestamp: $(date +"%Y-%m-%d %H:%M:%S%z")"
8093
echo "- hostname: $(hostname)"
81-
82-
if command -v Rscript &> /dev/null; then
83-
>&2 echo "ERROR: 'Rscript' could not be found. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_openlava, resources = list(modules = c(\"r\")))'. The search PATH for 'Rscript' was ${PATH}"
94+
echo "- Rscript call: <%= rscript_call %>"
95+
if ! command -v <%= rscript[1] %> &> /dev/null; then
96+
>&2 echo "ERROR: Argument 'resources' specifies a non-existing 'Rscript' launch command: <%= rscript[1] %>. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_slurm, resources = list(modules = c(\"r\")))'. The search PATH for '%<= rscript[1] %>' was ${PATH}"
8497
exit 1
8598
fi
86-
87-
echo "- Rscript path: $(which Rscript)"
88-
echo "- Rscript version: $(Rscript --version)"
89-
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
99+
echo "- Rscript version: $(<%= paste(rscript, collapse = " ") %> --version)"
100+
echo "- Rscript library paths: $(<%= rscript_call %> -e "cat(shQuote(.libPaths()), sep = ' ')")"
90101
echo
91102

92103

93104
## Launch R and evaluate the batchtools R job
94-
echo "Rscript -e 'batchtools::doJobCollection()' ..."
105+
echo "Calling 'batchtools::doJobCollection()' ..."
95106
echo "- job name: '<%= job.name %>'"
96107
echo "- job log file: '<%= log.file %>'"
97108
echo "- job uri: '<%= uri %>'"
98-
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
109+
<%= rscript_call %> -e 'batchtools::doJobCollection("<%= uri %>")'
99110
res=$?
100111
echo " - exit code: ${res}"
101-
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
112+
echo "Calling 'batchtools::doJobCollection()' ... done"
102113
echo
103114

104115
<% if (details) { %>

‎inst/templates/sge.tmpl‎

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
## as the working directory
2121
#$ -cwd
2222

23-
## Resources needed:
23+
## Resources needed
2424
<%
2525
## Should scheduler "details" be seen?
2626
details <- isTRUE(resources[["details"]])
@@ -37,15 +37,28 @@
3737
## Environment modules specifications
3838
modules <- resources[["modules"]]
3939
resources[["modules"]] <- NULL
40-
40+
41+
## Custom "Rscript" command and Rscript arguments
42+
rscript <- resources[["rscript"]]
43+
if (is.null(rscript)) {
44+
rscript <- "Rscript"
45+
} else {
46+
if (length(rscript) == 0 || !nzchar(rscript)[1]) stop("Argument 'resources' specifies an empty 'rscript' field")
47+
rscript[rscript == "*"] <- "Rscript"
48+
}
49+
resources[["rscript"]] <- NULL
50+
rscript_args <- resources[["rscript_args"]]
51+
resources[["rscript_args"]] <- NULL
52+
rscript_call <- paste(c(rscript, rscript_args), collapse = " ")
53+
4154
## As-is resource specifications
4255
job_declarations <- resources[["asis"]]
4356
resources[["asis"]] <- NULL
44-
45-
## Remaining resources are assumed to be of type '-l <key>=<value>'
57+
58+
## Remaining resources are assumed to be of type '<key>=<value>'
4659
opts <- unlist(resources, use.names = TRUE)
47-
opts <- sprintf("-l %s=%s", names(opts), opts)
48-
job_declarations <- sprintf("#$ %s", c(job_declarations, opts))
60+
opts <- sprintf("%s=%s", names(opts), opts)
61+
job_declarations <- sprintf("#$ %s", c(job_declarations, sprintf("-l %s", opts)))
4962
writeLines(job_declarations)
5063
%>
5164

@@ -87,27 +100,25 @@ fi
87100
echo "Session information:"
88101
echo "- timestamp: $(date +"%Y-%m-%d %H:%M:%S%z")"
89102
echo "- hostname: $(hostname)"
90-
91-
if command -v Rscript &> /dev/null; then
92-
>&2 echo "ERROR: 'Rscript' could not be found. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_sge, resources = list(modules = c(\"r\")))'. The search PATH for 'Rscript' was ${PATH}"
103+
echo "- Rscript call: <%= rscript_call %>"
104+
if ! command -v <%= rscript[1] %> &> /dev/null; then
105+
>&2 echo "ERROR: Argument 'resources' specifies a non-existing 'Rscript' launch command: <%= rscript[1] %>. Maybe you need to specify which environment modules to load in the 'resources' argument, e.g. 'plan(future.batchtools::batchtools_slurm, resources = list(modules = c(\"r\")))'. The search PATH for '%<= rscript[1] %>' was ${PATH}"
93106
exit 1
94107
fi
95-
96-
echo "- Rscript path: $(which Rscript)"
97-
echo "- Rscript version: $(Rscript --version)"
98-
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
108+
echo "- Rscript version: $(<%= paste(rscript, collapse = " ") %> --version)"
109+
echo "- Rscript library paths: $(<%= rscript_call %> -e "cat(shQuote(.libPaths()), sep = ' ')")"
99110
echo
100111

101112

102113
## Launch R and evaluate the batchtools R job
103-
echo "Rscript -e 'batchtools::doJobCollection()' ..."
114+
echo "Calling 'batchtools::doJobCollection()' ..."
104115
echo "- job name: '<%= job.name %>'"
105116
echo "- job log file: '<%= log.file %>'"
106117
echo "- job uri: '<%= uri %>'"
107-
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
118+
<%= rscript_call %> -e 'batchtools::doJobCollection("<%= uri %>")'
108119
res=$?
109120
echo " - exit code: ${res}"
110-
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
121+
echo "Calling 'batchtools::doJobCollection()' ... done"
111122
echo
112123

113124
<% if (details) { %>

0 commit comments

Comments
 (0)