Skip to content

Commit f3fb893

Browse files
HenrikBengtssonHenrikBengtsson
committed
Adjust 'R CMD check' templates
1 parent d369011 commit f3fb893

File tree

6 files changed

+345
-44
lines changed

6 files changed

+345
-44
lines changed

‎DESCRIPTION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Package: future.batchtools
2-
Version: 0.12.2-9980
2+
Version: 0.12.2-9981
33
Depends:
44
R (>= 3.2.0),
55
parallelly,

‎inst/templates-for-R_CMD_check/batchtools.sge.tmpl‎

Lines changed: 106 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,125 @@
1-
#!/usr/bin/env bash
1+
#!/bin/bash
2+
######################################################################
3+
# A batchtools launch script template for SGE
4+
#
5+
# Author: Henrik Bengtsson
6+
######################################################################
7+
## Shell
8+
#$ -S /bin/bash
29

3-
## Job name:
4-
#$ -N fbt-<%= job.hash %>
10+
## Job name
11+
#$ -N <%= job.name %>
512

6-
## Merge standard error and output:
7-
#$ -j y
8-
9-
## Direct streams to logfile:
13+
## Direct streams to logfile
1014
#$ -o <%= log.file %>
1115

12-
## Email on abort (a) and termination (e), but not when starting (b)
13-
#PBS -m ae
16+
## Merge standard error and output
17+
#$ -j y
1418

1519
## Tell the queue system to use the current directory
1620
## as the working directory
17-
## -cwd
18-
19-
## Mirror environment variables
20-
#$ -V
21+
#$ -cwd
2122

2223
## Resources needed:
23-
<% if (exists("resources", mode = "list") && length(resources) > 0) {
24-
if (isTRUE(getOption("future.debug"))) {
25-
R.utils::mcat("resources:")
26-
R.utils::mstr(resources)
27-
}
28-
cat(sprintf("#$ %s\n", resources[["custom"]]))
24+
<%
25+
## As-is resource specifications
26+
job_declarations <- resources[["asis"]]
27+
resources[["asis"]] <- NULL
28+
29+
## Should scheduler "details" be seen?
30+
details <- isTRUE(resources[["details"]])
31+
resources[["details"]] <- NULL
32+
33+
## Shell "startup" code to evaluate
34+
startup <- resources[["startup"]]
35+
resources[["startup"]] <- NULL
36+
37+
## Shell "shutdown" code to evaluate
38+
shutdown <- resources[["shutdown"]]
39+
resources[["shutdown"]] <- NULL
40+
41+
## Environment modules specifications
42+
modules <- resources[["modules"]]
43+
resources[["modules"]] <- NULL
44+
45+
## SPECIAL: For R CMD check package testing on HPC environments, which
46+
## typically uses a temporary working directory that is local, we force
47+
## it to use HPC-wide working directory
48+
path <- tools::R_user_dir("future.batchtools", "cache")
49+
path <- file.path(path, "R_CMD_check")
50+
if (!utils::file_test("-d", path)) dir.create(path, recursive = TRUE)
51+
job_declarations <- c(job_declarations, sprintf("-d %s", path))
52+
53+
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
54+
## by R CMD check is no longer found
55+
startup <- c(startup, "export R_TESTS=")
56+
57+
## Remaining resources are assumed to be of type '-l <key>=<value>'
58+
opts <- unlist(resources, use.names = TRUE)
59+
opts <- sprintf("-l %s=%s", names(opts), opts)
60+
job_declarations <- sprintf("#$ %s", c(job_declarations, opts))
61+
writeLines(job_declarations)
62+
%>
63+
64+
## Bash settings
65+
set -e # exit on error
66+
set -u # error on unset variables
67+
set -o pipefail # fail a pipeline if any command fails
68+
trap 'echo "ERROR: future.batchtools job script failed on line $LINENO" >&2; exit 1' ERR
69+
70+
echo "Batchtools information:"
71+
echo "- job name: '<%= job.name %>'"
72+
echo "- job log file: '<%= log.file %>'"
73+
echo
74+
75+
<% if (length(startup) > 0) {
76+
writeLines(startup)
77+
} %>
78+
79+
echo "Load environment modules:"
80+
<% if (length(modules) > 0) {
81+
writeLines(c(sprintf("module load %s", modules), "module list"))
2982
} %>
3083

31-
## UCSF HPC Pilot specific
32-
#$ -S /bin/bash # Required
33-
#$ -l mem_free=1G # Memory usage, required. Note that this is per slot
34-
#$ -R yes # SGE host reservation, highly recommended
84+
echo "Session information:"
85+
echo "- timestamp: $(date)"
86+
echo "- hostname: $(hostname)"
87+
echo "- Rscript path: $(which Rscript)"
88+
echo "- Rscript version: $(Rscript --version)"
89+
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
90+
echo
3591

36-
## SPECIAL: For R CMD check package testing on HPC environments, which
37-
## typically uses a temporary working directory that is local, we force
38-
## it to use HPC-wide working directory
39-
#$ -wd ~/tmp
92+
echo "Job submission declarations:"
93+
<%
94+
writeLines(sprintf("echo '%s'", job_declarations))
95+
%>
4096

41-
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
42-
## by R CMD check is no longer found
43-
export R_TESTS=
97+
<% if (details) { %>
98+
if command -v qstat > /dev/null; then
99+
echo "Job information:"
100+
qstat -j "${JOB_ID}"
101+
echo
102+
fi
103+
<% } %>
44104

45-
# For troubleshooting if there are errors
46-
date
47-
hostname
48-
which Rscript
49-
Rscript --version
50-
Rscript -e ".libPaths()"
51105

106+
## Launch R and evaluate the batchtools R job
52107
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
53108
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
109+
res=$?
110+
echo " - exit code: ${res}"
54111
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
55112

56-
## For troubleshooting if there are errors
57-
## https://github.com/UCSF-HPC/pilot-testing/issues/1
58-
qstat -j $JOB_ID
113+
<% if (details) { %>
114+
if command -v qstat > /dev/null; then
115+
echo "Job summary:"
116+
qstat -j "${JOB_ID}"
117+
fi
118+
<% } %>
119+
120+
<% if (length(shutdown) > 0) {
121+
writeLines(shutdown)
122+
} %>
123+
124+
## Relay the exit code from Rscript
125+
exit "${res}"

‎inst/templates-for-R_CMD_check/batchtools.slurm.tmpl‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
#!/bin/bash
2+
######################################################################
3+
# A batchtools launch script template for Slurm
4+
#
5+
# Author: Henrik Bengtsson
6+
######################################################################
7+
8+
## Job name
9+
#SBATCH --job-name=<%= job.name %>
10+
## Direct streams to logfile
11+
#SBATCH --output=<%= log.file %>
12+
13+
## Resources needed:
14+
<%
15+
## As-is resource specifications
16+
job_declarations <- resources[["asis"]]
17+
resources[["asis"]] <- NULL
18+
19+
## Shell "details" code to evaluate
20+
details <- isTRUE(resources[["details"]])
21+
resources[["details"]] <- NULL
22+
23+
## Shell "startup" code to evaluate
24+
startup <- resources[["startup"]]
25+
resources[["startup"]] <- NULL
26+
27+
## Shell "shutdown" code to evaluate
28+
shutdown <- resources[["shutdown"]]
29+
resources[["shutdown"]] <- NULL
30+
31+
## Environment modules specifications
32+
modules <- resources[["modules"]]
33+
resources[["modules"]] <- NULL
34+
35+
## SPECIAL: For R CMD check package testing on HPC environments, which
36+
## typically uses a temporary working directory that is local, we force
37+
## it to use HPC-wide working directory
38+
path <- tools::R_user_dir("future.batchtools", "cache")
39+
path <- file.path(path, "R_CMD_check")
40+
if (!utils::file_test("-d", path)) dir.create(path, recursive = TRUE)
41+
job_declarations <- c(job_declarations, sprintf("--chdir=%s", path))
42+
43+
## SPECIAL: Since we change working directory, the 'startup.Rs' file used
44+
## by R CMD check is no longer found
45+
startup <- c(startup, "export R_TESTS=")
46+
47+
## Remaining resources are assumed to be of type '--<key>=<value>'
48+
opts <- unlist(resources, use.names = TRUE)
49+
opts <- sprintf("--%s=%s", names(opts), opts)
50+
job_declarations <- sprintf("#SBATCH %s", c(job_declarations, opts))
51+
writeLines(job_declarations)
52+
%>
53+
54+
## Bash settings
55+
set -e # exit on error
56+
set -u # error on unset variables
57+
set -o pipefail # fail a pipeline if any command fails
58+
trap 'echo "ERROR: future.batchtools job script failed on line $LINENO" >&2; exit 1' ERR
59+
60+
echo "Batchtools information:"
61+
echo "- job name: '<%= job.name %>'"
62+
echo "- job log file: '<%= log.file %>'"
63+
echo
64+
65+
<% if (length(startup) > 0) {
66+
writeLines(startup)
67+
} %>
68+
69+
echo "Load environment modules:"
70+
<% if (length(modules) > 0) {
71+
writeLines(c(sprintf("module load %s", modules), "module list"))
72+
} %>
73+
74+
echo "Session information:"
75+
echo "- timestamp: $(date)"
76+
echo "- hostname: $(hostname)"
77+
echo "- Rscript path: $(which Rscript)"
78+
echo "- Rscript version: $(Rscript --version)"
79+
echo "- Rscript library paths: $(Rscript -e "cat(shQuote(.libPaths()), sep = ' ')")"
80+
echo
81+
82+
echo "Job submission declarations:"
83+
<%
84+
writeLines(sprintf("echo '%s'", job_declarations))
85+
%>
86+
87+
<% if (details) { %>
88+
if command -v scontrol > /dev/null; then
89+
echo "Slurm job information:"
90+
scontrol show job "${SLURM_JOB_ID}"
91+
echo
92+
fi
93+
<% } %>
94+
95+
## Launch R and evaluate the batchtools R job
96+
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
97+
Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
98+
res=$?
99+
echo " - exit code: ${res}"
100+
echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
101+
102+
<% if (details) { %>
103+
if command -v sstat > /dev/null; then
104+
echo "Job summary:"
105+
sstat --format="JobID,AveCPU,MaxRSS,MaxPages,MaxDiskRead,MaxDiskWrite" --allsteps --jobs="${SLURM_JOB_ID}"
106+
fi
107+
<% } %>
108+
109+
<% if (length(shutdown) > 0) {
110+
writeLines(shutdown)
111+
} %>
112+
113+
## Relay the exit code from Rscript
114+
exit "${res}"

0 commit comments

Comments
 (0)