Skip to content

Commit dafd107

Browse files
HenrikBengtssonHenrikBengtsson
committed
TESTS: Make it check via SGE, Torque/PBS and Slurm schedulers, if specified
1 parent 1d83b12 commit dafd107

File tree

4 files changed

+114
-117
lines changed

4 files changed

+114
-117
lines changed

‎DESCRIPTION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Package: future.batchtools
2-
Version: 0.12.2-9984
2+
Version: 0.12.2-9985
33
Depends:
44
R (>= 3.2.0),
55
parallelly,

‎inst/templates-for-R_CMD_check/batchtools.sge.tmpl‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
## As-is resource specifications
2626
job_declarations <- resources[["asis"]]
2727
resources[["asis"]] <- NULL
28-
28+
2929
## Should scheduler "details" be seen?
3030
details <- isTRUE(resources[["details"]])
3131
resources[["details"]] <- NULL
@@ -41,7 +41,7 @@
4141
## Environment modules specifications
4242
modules <- resources[["modules"]]
4343
resources[["modules"]] <- NULL
44-
44+
4545
## SPECIAL: For R CMD check package testing on HPC environments, which
4646
## typically uses a temporary working directory that is local, we force
4747
## it to use HPC-wide working directory
@@ -52,8 +52,17 @@
5252

5353
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
5454
## by R CMD check is no longer found
55-
startup <- c(startup, "export R_TESTS=")
55+
env <- Sys.getenv("R_TESTS", NA_character_)
56+
if (!is.na(env)) startup <- c(startup, sprintf("export R_TESTS=%s", normalizePath(env)))
5657

58+
## SPECIAL: Use the 'Rscript' on PATH when processing this template; not
59+
## the dummy one injected by 'R CMD check'
60+
startup <- c(startup, sprintf('export PATH="%s:${PATH}"', R.home("bin")))
61+
62+
## SPECIAL: Use the 'R_LIBS_USER' at compile time, because of 'R CMD check'
63+
libs <- dirname(unique(vapply(c("future.batchtools", "batchtools"), FUN = find.package, FUN.VALUE = "")))
64+
startup <- c(startup, sprintf("export R_LIBS_USER=%s", paste(libs, collapse = .Platform[["path.sep"]])))
65+
5766
## Remaining resources are assumed to be of type '-l <key>=<value>'
5867
opts <- unlist(resources, use.names = TRUE)
5968
opts <- sprintf("-l %s=%s", names(opts), opts)
@@ -67,18 +76,33 @@ set -u # error on unset variables
6776
set -o pipefail # fail a pipeline if any command fails
6877
trap 'echo "ERROR: future.batchtools job script failed on line $LINENO" >&2; exit 1' ERR
6978

70-
echo "Batchtools information:"
71-
echo "- job name: '<%= job.name %>'"
72-
echo "- job log file: '<%= log.file %>'"
73-
echo
79+
<% if (length(job_declarations) > 0) {
80+
writeLines(c(
81+
"echo 'Job submission declarations:'",
82+
sprintf("echo '%s'", job_declarations),
83+
"echo"
84+
))
85+
} %>
86+
87+
<% if (details) { %>
88+
if command -v qstat > /dev/null; then
89+
echo "Job information:"
90+
qstat -j "${JOB_ID}"
91+
echo
92+
fi
93+
<% } %>
7494

7595
<% if (length(startup) > 0) {
7696
writeLines(startup)
7797
} %>
7898

79-
echo "Load environment modules:"
8099
<% if (length(modules) > 0) {
81-
writeLines(c(sprintf("module load %s", modules), "module list"))
100+
writeLines(c(
101+
"echo 'Load environment modules:'",
102+
sprintf("echo '- modules: %s'", paste(modules, collapse = ", ")),
103+
sprintf("module load %s", paste(modules, collapse = " ")),
104+
"module list"
105+
))
82106
} %>
83107

84108
echo "Session information:"
@@ -89,26 +113,17 @@ echo "- Rscript version: $(Rscript --version)"
89113
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
90114
echo
91115

92-
echo "Job submission declarations:"
93-
<%
94-
writeLines(sprintf("echo '%s'", job_declarations))
95-
%>
96-
97-
<% if (details) { %>
98-
if command -v qstat > /dev/null; then
99-
echo "Job information:"
100-
qstat -j "${JOB_ID}"
101-
echo
102-
fi
103-
<% } %>
104-
105116

106117
## Launch R and evaluate the batchtools R job
107-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
118+
echo "Rscript -e 'batchtools::doJobCollection()' ..."
119+
echo "- job name: '<%= job.name %>'"
120+
echo "- job log file: '<%= log.file %>'"
121+
echo "- job uri: '<%= uri %>'"
108122
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
109123
res=$?
110124
echo " - exit code: ${res}"
111-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
125+
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
126+
echo
112127

113128
<% if (details) { %>
114129
if command -v qstat > /dev/null; then

‎inst/templates-for-R_CMD_check/batchtools.slurm.tmpl‎

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,17 @@
4242

4343
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
4444
## by R CMD check is no longer found
45-
startup <- c(startup, "export R_TESTS=")
45+
env <- Sys.getenv("R_TESTS", NA_character_)
46+
if (!is.na(env)) startup <- c(startup, sprintf("export R_TESTS=%s", normalizePath(env)))
4647

48+
## SPECIAL: Use the 'Rscript' on PATH when processing this template; not
49+
## the dummy one injected by 'R CMD check'
50+
startup <- c(startup, sprintf('export PATH="%s:${PATH}"', R.home("bin")))
51+
52+
## SPECIAL: Use the 'R_LIBS_USER' at compile time, because of 'R CMD check'
53+
libs <- dirname(unique(vapply(c("future.batchtools", "batchtools"), FUN = find.package, FUN.VALUE = "")))
54+
startup <- c(startup, sprintf("export R_LIBS_USER=%s", paste(libs, collapse = .Platform[["path.sep"]])))
55+
4756
## Remaining resources are assumed to be of type '--<key>=<value>'
4857
opts <- unlist(resources, use.names = TRUE)
4958
opts <- sprintf("--%s=%s", names(opts), opts)
@@ -57,18 +66,33 @@ set -u # error on unset variables
5766
set -o pipefail # fail a pipeline if any command fails
5867
trap 'echo "ERROR: future.batchtools job script failed on line $LINENO" >&2; exit 1' ERR
5968

60-
echo "Batchtools information:"
61-
echo "- job name: '<%= job.name %>'"
62-
echo "- job log file: '<%= log.file %>'"
63-
echo
69+
<% if (length(job_declarations) > 0) {
70+
writeLines(c(
71+
"echo 'Job submission declarations:'",
72+
sprintf("echo '%s'", job_declarations),
73+
"echo"
74+
))
75+
} %>
76+
77+
<% if (details) { %>
78+
if command -v scontrol > /dev/null; then
79+
echo "Job information:"
80+
scontrol show job "${SLURM_JOB_ID}"
81+
echo
82+
fi
83+
<% } %>
6484

6585
<% if (length(startup) > 0) {
6686
writeLines(startup)
6787
} %>
6888

69-
echo "Load environment modules:"
7089
<% if (length(modules) > 0) {
71-
writeLines(c(sprintf("module load %s", modules), "module list"))
90+
writeLines(c(
91+
"echo 'Load environment modules:'",
92+
sprintf("echo '- modules: %s'", paste(modules, collapse = ", ")),
93+
sprintf("module load %s", paste(modules, collapse = " ")),
94+
"module list"
95+
))
7296
} %>
7397

7498
echo "Session information:"
@@ -79,25 +103,16 @@ echo "- Rscript version: $(Rscript --version)"
79103
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
80104
echo
81105

82-
echo "Job submission declarations:"
83-
<%
84-
writeLines(sprintf("echo '%s'", job_declarations))
85-
%>
86-
87-
<% if (details) { %>
88-
if command -v scontrol > /dev/null; then
89-
echo "Slurm job information:"
90-
scontrol show job "${SLURM_JOB_ID}"
91-
echo
92-
fi
93-
<% } %>
94-
95106
## Launch R and evaluate the batchtools R job
96-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
107+
echo "Rscript -e 'batchtools::doJobCollection()' ..."
108+
echo "- job name: '<%= job.name %>'"
109+
echo "- job log file: '<%= log.file %>'"
110+
echo "- job uri: '<%= uri %>'"
97111
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
98112
res=$?
99113
echo " - exit code: ${res}"
100-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
114+
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
115+
echo
101116

102117
<% if (details) { %>
103118
if command -v sstat > /dev/null; then

‎inst/templates-for-R_CMD_check/batchtools.torque.tmpl‎

Lines changed: 37 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,17 @@
4646

4747
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
4848
## by R CMD check is no longer found
49-
startup <- c(startup, "export R_TESTS=")
49+
env <- Sys.getenv("R_TESTS", NA_character_)
50+
if (!is.na(env)) startup <- c(startup, sprintf("export R_TESTS=%s", normalizePath(env)))
5051

52+
## SPECIAL: Use the 'Rscript' on PATH when processing this template; not
53+
## the dummy one injected by 'R CMD check'
54+
startup <- c(startup, sprintf('export PATH="%s:${PATH}"', R.home("bin")))
55+
56+
## SPECIAL: Use the 'R_LIBS_USER' at compile time, because of 'R CMD check'
57+
libs <- dirname(unique(vapply(c("future.batchtools", "batchtools"), FUN = find.package, FUN.VALUE = "")))
58+
startup <- c(startup, sprintf("export R_LIBS_USER=%s", paste(libs, collapse = .Platform[["path.sep"]])))
59+
5160
## Remaining resources are assumed to be of type '-l <key>=<value>'
5261
opts <- unlist(resources, use.names = TRUE)
5362
opts <- sprintf("-l %s=%s", names(opts), opts)
@@ -61,18 +70,33 @@ set -u # error on unset variables
6170
set -o pipefail # fail a pipeline if any command fails
6271
trap 'echo "ERROR: future.batchtools job script failed on line $LINENO" >&2; exit 1' ERR
6372

64-
echo "Batchtools information:"
65-
echo "- job name: '<%= job.name %>'"
66-
echo "- job log file: '<%= log.file %>'"
67-
echo
73+
<% if (length(job_declarations) > 0) {
74+
writeLines(c(
75+
"echo 'Job submission declarations:'",
76+
sprintf("echo '%s'", job_declarations),
77+
"echo"
78+
))
79+
} %>
80+
81+
<% if (details) { %>
82+
if command -v qstat > /dev/null; then
83+
echo "Job information:"
84+
qstat -f "${PBS_JOBID}"
85+
echo
86+
fi
87+
<% } %>
6888

6989
<% if (length(startup) > 0) {
7090
writeLines(startup)
7191
} %>
7292

73-
echo "Load environment modules:"
7493
<% if (length(modules) > 0) {
75-
writeLines(c(sprintf("module load %s", modules), "module list"))
94+
writeLines(c(
95+
"echo 'Load environment modules:'",
96+
sprintf("echo '- modules: %s'", paste(modules, collapse = ", ")),
97+
sprintf("module load %s", paste(modules, collapse = " ")),
98+
"module list"
99+
))
76100
} %>
77101

78102
echo "Session information:"
@@ -83,25 +107,16 @@ echo "- Rscript version: $(Rscript --version)"
83107
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
84108
echo
85109

86-
echo "Job submission declarations:"
87-
<% if (length(modules) > 0) {
88-
writeLines(c(sprintf("module load %s", modules), "module list"))
89-
} %>
90-
91-
<% if (details) { %>
92-
if command -v qstat > /dev/null; then
93-
echo "Job information:"
94-
qstat -f "${PBS_JOBID}"
95-
echo
96-
fi
97-
<% } %>
98-
99110
## Launch R and evaluate the batchtools R job
100-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
111+
echo "Rscript -e 'batchtools::doJobCollection()' ..."
112+
echo "- job name: '<%= job.name %>'"
113+
echo "- job log file: '<%= log.file %>'"
114+
echo "- job uri: '<%= uri %>'"
101115
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
102116
res=$?
103117
echo " - exit code: ${res}"
104-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
118+
echo "Rscript -e 'batchtools::doJobCollection()' ... done"
119+
echo
105120

106121
<% if (details) { %>
107122
if command -v qstat > /dev/null; then
@@ -116,51 +131,3 @@ fi
116131

117132
## Relay the exit code from Rscript
118133
exit "${res}"
119-
#!/usr/bin/env bash
120-
121-
## Job name:
122-
#PBS -N fbt-<%= job.hash %>
123-
124-
## Merge standard error and output:
125-
#PBS -j oe
126-
127-
## Direct streams to logfile:
128-
#PBS -o <%= log.file %>
129-
130-
## Email on abort (a) and termination (e), but not when starting (b)
131-
#PBS -m ae
132-
133-
## Mirror environment variables
134-
#PBS -V
135-
136-
## Resources needed:
137-
<% if (exists("resources", mode = "list") && length(resources) > 0) {
138-
if (isTRUE(getOption("future.debug"))) {
139-
R.utils::mcat("resources:")
140-
R.utils::mstr(resources)
141-
}
142-
opts <- unlist(resources, use.names = TRUE)
143-
opts <- sprintf("%s=%s", names(opts), opts)
144-
opts <- paste(opts, collapse = ",") %>
145-
#PBS -l <%= opts %>
146-
<% } %>
147-
148-
## SPECIAL: For R CMD check package testing on HPC environments, which
149-
## typically uses a temporary working directory that is local, we force
150-
## it to use HPC-wide working directory
151-
#PBS -d <%= normalizePath(file.path("~", "tmp")) %>
152-
153-
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
154-
## by R CMD check is no longer found
155-
export R_TESTS=
156-
157-
# For troubleshooting if there are errors
158-
date
159-
hostname
160-
which Rscript
161-
Rscript --version
162-
Rscript -e ".libPaths()"
163-
164-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
165-
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
166-
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"

0 commit comments

Comments
 (0)