Built-in templates: Add support for resources[["details"]] [#26]

Author: HenrikBengtsson

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: future.batchtools
-Version: 0.12.2-9979
+Version: 0.12.2-9980
 Depends:
     R (>= 3.2.0),
     parallelly,

NEWS.md

@@ -33,19 +33,24 @@
    - Add built-in template job scripts for LSF and OpenLava.
 
    - All built-in template job scripts support "as-is" resource
-     specifications via `resources[["asis"]]`.
+     specifications via character vector `resources[["asis"]]`.
 
    - All built-in template job scripts support "environment module"
-     resource specifications via `resources[["modules"]]`. When
-     specified, corresponding `module load <name>` entries are
-     injected to the generated job script.
+     resource specifications via character vector
+     `resources[["modules"]]`. When specified, corresponding `module
+     load <name>` entries are injected to the generated job script.
 
    - All built-in template job scripts support "startup" and
-     "shutdown" code resource specifications via
+     "shutdown" code resource specifications via character vectors
      `resources[["startup"]]` and `resources[["shutdown"]]`. When
      specified, corresponding lines are injected in the generated job
      script and the beginning and end, respectively.
-     
+
+   - All built-in template job scripts support a "details" resource
+     specification via logical scalar `resources[["details"]]`. If
+     TRUE, scheduler job details are outputted to the log files at the
+     beginning and job summaries at the end.
+
    - All built-in template job scripts, which are written in Bash,
      error and exit early, but setting more strict Bash options. This
      should help with any troubleshooting, especially when getting

R/BatchtoolsFutureBackend-class.R

@@ -27,8 +27,20 @@
 #' The default file is the one found by [batchtools::findConfFile()], if any.
 #'
 #' @param resources (optional) A named list passed to the \pkg{batchtools}
-#' job-script template as variable `resources`.  See Section 'Resources'
-#' in [batchtools::submitJobs()] more details.
+#' job-script template as variable `resources`. This is based on how
+#' [batchtools::submitJobs()] works, with the exception for specially
+#' reserved names defined by the \pkg{future.batchtools} package;
+#' * `resources[["asis"]]` is a character vector that are passed as-is to
+#'   the job script and are injected as job resource declarations.
+#' * `resources[["modules"]]` is character vector of Linux environment
+#'   modules to be loaded.
+#' * `resources[["startup"]]` and `resources[["shutdown"]]` are character
+#'   vectors of shell code to be injected to the job script as-is.
+#' * `resources[["details"]]`, if TRUE, results in the job script outputting
+#'   job details and job summaries at the beginning and then end.
+#' * All remaining `resources` named elements are injected as named resource
+#'   specification for the scheduler.
+#'
 #'
 #' @param delete Controls if and when the batchtools job registry folder is
 #' deleted.

R/batchtools_lsf.R

@@ -41,7 +41,9 @@ BatchtoolsLsfFutureBackend <- function(...) {
 #' # Submit to the 'freecycle' queue.
 #' plan(future.batchtools::batchtools_lsf, resources = list(
 #'   W = "00:10:00", M = "400",
-#'   asis = c("-n 4", "-R 'span[hosts=1]'", "-q freecycle")
+#'   asis = c("-n 4", "-R 'span[hosts=1]'", "-q freecycle"),
+#'   modules = c("r", "jags"),
+#'   details = TRUE
 #' ))
 #'
 #' f <- future({

R/batchtools_openlava.R

@@ -41,7 +41,9 @@ BatchtoolsOpenLavaFutureBackend <- function(...) {
 #' # Submit to the 'freecycle' queue.
 #' plan(future.batchtools::batchtools_openlava, resources = list(
 #'   W = "00:10:00", M = "400",
-#'   asis = c("-n 4", "-R 'span[hosts=1]'", "-q freecycle")
+#'   asis = c("-n 4", "-R 'span[hosts=1]'", "-q freecycle"),
+#'   modules = c("r", "jags"),
+#'   details = TRUE
 #' ))
 #'
 #' f <- future({

R/batchtools_sge.R

@@ -54,7 +54,9 @@ print.BatchtoolsSGEFutureBackend <- function(x, ...) {
 #' # Submit to the 'freecycle' queue.
 #' plan(future.batchtools::batchtools_sge, resources = list(
 #'   h_rt = "00:10:00", mem_free = "100M",  ## memory is per process
-#'   asis = c("-pe smp 4", "-q freecycle.q")
+#'   asis = c("-pe smp 4", "-q freecycle.q"),
+#'   modules = c("r", "jags"),
+#'   details = TRUE
 #' ))
 #'
 #' f <- future({

R/batchtools_slurm.R

@@ -58,7 +58,9 @@ print.BatchtoolsSlurmFutureBackend <- function(x, ...) {
 #' # Submit to the 'freecycle' partition.
 #' plan(future.batchtools::batchtools_slurm, resources = list(
 #'   time = "00:10:00", mem = "400M",
-#'   asis = c("--nodes=1", "--ntasks=4", "--partition=freecycle")
+#'   asis = c("--nodes=1", "--ntasks=4", "--partition=freecycle"),
+#'   modules = c("r", "jags"),
+#'   details = TRUE
 #' ))
 #'
 #' f <- future({

R/batchtools_torque.R

@@ -40,7 +40,9 @@ BatchtoolsTorqueFutureBackend <- function(...) {
 #' # Submit to the 'freecycle' queue.
 #' plan(future.batchtools::batchtools_torque, resources = list(
 #'   walltime = "00:10:00", mem = "100mb",  ## memory is per process
-#'   asis = c("-l nodes=1:ppn=4", "-q freecycle")
+#'   asis = c("-l nodes=1:ppn=4", "-q freecycle"),
+#'   modules = c("r", "jags"),
+#'   details = TRUE
 #' ))
 #'
 #' f <- future({

inst/templates/lsf.tmpl

@@ -17,6 +17,10 @@
   job_declarations <- resources[["asis"]]
   resources[["asis"]] <- NULL
 
+  ## Should scheduler "details" be seen?
+  details <- isTRUE(resources[["details"]])
+  resources[["details"]] <- NULL
+
   ## Shell "startup" code to evaluate
   startup <- resources[["startup"]]
   resources[["startup"]] <- NULL
@@ -70,11 +74,14 @@ echo "Job submission declarations:"
   writeLines(sprintf("echo '%s'", job_declarations))
 %>
 
+<% if (details) { %>
 if command -v bjobs > /dev/null; then
   echo "Job information:"
   bjobs -l "${LSB_JOBID}"
   echo
 fi    
+<% } %>
+
 
 ## Launch R and evaluate the batchtools R job
 echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
@@ -83,11 +90,12 @@ res=$?
 echo " - exit code: ${res}"
 echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
 
-## End-of-job summary
+<% if (details) { %>
 if command -v bjobs > /dev/null; then
   echo "Job summary:"
   bjobs -l "${LSB_JOBID}"
 fi
+<% } %>
 
 <% if (length(shutdown) > 0) {
   ## Inject optional shell code

inst/templates/openlava.tmpl

@@ -17,6 +17,10 @@
   job_declarations <- resources[["asis"]]
   resources[["asis"]] <- NULL
 
+  ## Should scheduler "details" be seen?
+  details <- isTRUE(resources[["details"]])
+  resources[["details"]] <- NULL
+
   ## Shell "startup" code to evaluate
   startup <- resources[["startup"]]
   resources[["startup"]] <- NULL
@@ -70,11 +74,14 @@ echo "Job submission declarations:"
   writeLines(sprintf("echo '%s'", job_declarations))
 %>
 
+<% if (details) { %>
 if command -v bjobs > /dev/null; then
   echo "Job information:"
   bjobs -l "${LSB_JOBID}"
   echo
 fi    
+<% } %>
+
 
 ## Launch R and evaluate the batchtools R job
 echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ..."
@@ -83,11 +90,12 @@ res=$?
 echo " - exit code: ${res}"
 echo "Command: Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ... done"
 
-## End-of-job summary
+<% if (details) { %>
 if command -v bjobs > /dev/null; then
   echo "Job summary:"
   bjobs -l "${LSB_JOBID}"
 fi
+<% } %>
 
 <% if (length(shutdown) > 0) {
   ## Inject optional shell code