batchtools_slurm(): help + tweak template script

Author: HenrikBengtsson

R/batchtools_bash.R

@@ -20,7 +20,7 @@ BatchtoolsBashFutureBackend <- function(...,
 }
 
 
-#' A batchtools bash backend that resolves futures sequential via a Bash template script
+#' A batchtools bash backend that resolves futures sequentially via a Bash template script
 #'
 #' The `batchtools_bash` backend was added to illustrate how to write a
 #' custom \pkg{future.batchtools} backend that uses a templated job script.
@@ -37,7 +37,7 @@ BatchtoolsBashFutureBackend <- function(...,
 #' @param \ldots Not used.
 #'
 #' @details
-#' Batchtools bash futures uses \pkg{batchtools} cluster functions
+#' Batchtools bash futures use \pkg{batchtools} cluster functions
 #' created by [makeClusterFunctionsBash()] and requires that `bash` is
 #' installed on the current machine and the `timeout` command is available.
 #'

R/batchtools_slurm.R

@@ -0,0 +1,151 @@
+#' Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc.
+#'
+#' Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc. are
+#' asynchronous multiprocess futures that will be evaluated on a compute
+#' cluster via a job scheduler.
+#'
+#' @inheritParams BatchtoolsFutureBackend
+#'
+#' @param template (optional) A batchtools template file or a template string
+#' (in \pkg{brew} format).  If not specified, it is left to the
+#' \pkg{batchtools} package to locate such file using its search rules.
+#'
+#' @param type (character string) Type of job scheduler.
+#'
+#' @param \ldots Additional arguments passed to [BatchtoolsFutureBackend()].
+#'
+#' @return An object of class `BatchtoolsFutureBackend`.
+#'
+#' @details
+#' These type of batchtools futures rely on batchtools backends set
+#' up using the following \pkg{batchtools} functions:
+#'
+#'  * [batchtools::makeClusterFunctionsLSF()] for
+#'    [Load Sharing Facility (LSF)](https://en.wikipedia.org/wiki/Platform_LSF)
+#'  * [batchtools::makeClusterFunctionsOpenLava()] for
+#'    [OpenLava](https://en.wikipedia.org/wiki/OpenLava)
+#'  * [batchtools::makeClusterFunctionsSGE()] for
+#'    [Sun/Oracle Grid Engine (SGE)](https://en.wikipedia.org/wiki/Oracle_Grid_Engine)
+#'  * [batchtools::makeClusterFunctionsSlurm()] for
+#'    [Slurm](https://en.wikipedia.org/wiki/Slurm_Workload_Manager)
+#'  * [batchtools::makeClusterFunctionsTORQUE()] for
+#'    [TORQUE](https://en.wikipedia.org/wiki/TORQUE) / PBS
+#'
+#' @aliases BatchtoolsLsfFutureBackend BatchtoolsOpenLavaFutureBackend BatchtoolsSGEFutureBackend BatchtoolsSlurmFutureBackend BatchtoolsTorqueFutureBackend
+#' @aliases batchtools_lsf batchtools_openlava batchtools_sge batchtools_torque
+#'     
+#' @importFrom batchtools makeClusterFunctionsLSF
+#' @importFrom batchtools makeClusterFunctionsOpenLava
+#' @importFrom batchtools makeClusterFunctionsSGE
+#' @importFrom batchtools makeClusterFunctionsSlurm
+#' @importFrom batchtools makeClusterFunctionsTORQUE
+#' @export
+BatchtoolsTemplateFutureBackend <- function(..., template = NULL, type = c("lsf", "openlava", "sge", "slurm", "torque")) {
+  assert_no_positional_args_but_first()
+  type <- match.arg(type)
+
+
+  dotdotdot <- list(...)
+
+  make_cfs <- switch(type,
+    lsf      = makeClusterFunctionsLSF,
+    openlava = makeClusterFunctionsOpenLava,
+    sge      = makeClusterFunctionsSGE,
+    slurm    = makeClusterFunctionsSlurm,
+    torque   = makeClusterFunctionsTORQUE
+  )
+
+  make_cfs_formals <- formals(make_cfs)
+  
+  ## Get the default template?
+  if (is.null(template)) {
+    template <- make_cfs_formals$template
+  }
+
+  stop_if_not(is.character(template), length(template) == 1L,
+              !is.na(template), nzchar(template))
+
+  template <- find_template_file(template)
+
+  keep <- which(names(dotdotdot) %in% names(make_cfs_formals))
+  args <- c(list(template = template), dotdotdot[keep])
+  cluster.functions <- do.call(make_cfs, args = args)
+  attr(cluster.functions, "template") <- template
+
+  ## Drop used '...' arguments
+  if (length(keep) > 0) dotdotdot <- dotdotdot[-keep]
+
+  args <- dotdotdot
+  args[["cluster.functions"]] <- cluster.functions
+  
+  core <- do.call(BatchtoolsMultiprocessFutureBackend, args = args)
+  
+  core[["futureClasses"]] <- c("BatchtoolsTemplateFuture", "BatchtoolsMultiprocessFuture", core[["futureClasses"]])
+  core <- structure(core, class = c("BatchtoolsTemplateFutureBackend", class(core)))
+  core
+}
+
+
+#' @export
+BatchtoolsSlurmFutureBackend <- function(...) {
+  core <- BatchtoolsTemplateFutureBackend(..., type = "slurm")
+  core[["futureClasses"]] <- c("BatchtoolsSlurmFuture", core[["futureClasses"]])
+  core <- structure(core, class = c("BatchtoolsSlurmFutureBackend", class(core)))
+  core
+}
+
+
+#' A batchtools slurm backend resolves futures in parallel via a Slurm job scheduler
+#'
+#' @inheritParams BatchtoolsFutureBackend
+#' @inheritParams BatchtoolsTemplateFutureBackend
+#'
+#' @param template (optional) Name of job-script template to be searched
+#' for by [batchtools::findTemplateFile()]. If not found, it defaults to
+#' the `templates/slurm.tmpl` part of this package (see below).
+#'
+#' @param \ldots Not used.
+#'
+#' @details
+#' Batchtools slurm futures use \pkg{batchtools} cluster functions
+#' created by [batchtools::makeClusterFunctionsSlurm()], which requires
+#' that Slurm commands `sbatch`, `squeue`, and `scancel` are installed on
+#' the current machine.
+#'
+#' The default template script `templates/slurm.tmpl` can be found in:
+#'
+#' ```r
+#' system.file("templates", "slurm.tmpl", package = "future.batchtools")
+#' ```
+#'
+#' and comprise:
+#'
+#' `r paste(c("\x60\x60\x60bash", readLines("inst/templates/slurm.tmpl"), "\x60\x60\x60"), collapse = "\n")`
+#'
+#' @examplesIf interactive()
+#' # Limit runtime to 3 minutes and memory to 200 MiB per future
+#' plan(batchtools_slurm, resources = list(time = "00:03:00", mem = "200M"))
+#'
+#' message("Main process ID: ", Sys.getpid())
+#'
+#' f <- future(Sys.getpid())
+#' pid <- value(f)
+#' message("Worker process ID: ", pid)
+#' 
+#' @export
+batchtools_slurm <- function(...) {
+ stop("INTERNAL ERROR: The future.batchtools::batchtools_slurm() must never be called directly")
+}
+class(batchtools_slurm) <- c(
+  "batchtools_slurm", "batchtools_template",
+  "batchtools_multiprocess", "batchtools",
+  "multiprocess", "future", "function"
+)
+attr(batchtools_slurm, "tweakable") <- c(
+  "workers",
+  "finalize",
+  ## Arguments to batchtools::makeClusterFunctionsSlurm()
+  "array.jobs", "nodename", "scheduler.latency", "fs.latency"
+)
+attr(batchtools_slurm, "init") <- TRUE
+attr(batchtools_slurm, "factory") <- BatchtoolsSlurmFutureBackend

R/batchtools_template.R

@@ -32,7 +32,7 @@
 #'    [TORQUE](https://en.wikipedia.org/wiki/TORQUE) / PBS
 #'
 #' @aliases BatchtoolsLsfFutureBackend BatchtoolsOpenLavaFutureBackend BatchtoolsSGEFutureBackend BatchtoolsSlurmFutureBackend BatchtoolsTorqueFutureBackend
-#' @aliases batchtools_lsf batchtools_openlava batchtools_sge batchtools_slurm batchtools_torque
+#' @aliases batchtools_lsf batchtools_openlava batchtools_sge batchtools_torque
 #'     
 #' @importFrom batchtools makeClusterFunctionsLSF
 #' @importFrom batchtools makeClusterFunctionsOpenLava

inst/templates/bash.tmpl

@@ -1,6 +1,8 @@
 #!/bin/bash
 ######################################################################
 # A batchtools launch script template
+#
+# Author: Henrik Bengtsson 
 ######################################################################
 
 ## Maximum runtime?

inst/templates/slurm.tmpl

@@ -5,18 +5,21 @@
 # Author: Henrik Bengtsson 
 ######################################################################
 
+<% Slurm defaults
+defaults <- list(
+  nodes = 1,          # single-host processing
+   time = "00:05:00"  # 5-min runtime
+   mem  = "100M"      # 100 MiB memory
+)
+resources <- c(resources, defaults[setdiff(names(defaults), names(resources))])
+opts <- unlist(resources, use.names = TRUE)
+opts <- sprintf("--%s=%s", names(opts), opts)
+opts <- paste(opts, collapse = " ") %>
+%>
+
 #SBATCH --job-name=<%= job.name %>
 #SBATCH --output=<%= log.file %>
-#SBATCH --nodes=1
-#SBATCH --time=00:05:00
-
-## Resources needed:
-<% if (length(resources) > 0) {
-  opts <- unlist(resources, use.names = TRUE)
-  opts <- sprintf("--%s=%s", names(opts), opts)
-  opts <- paste(opts, collapse = " ") %>
 #SBATCH <%= opts %>
-<% } %>
 
-## Launch R and evaluated the batchtools R job
+## Launch R and evaluate the batchtools R job
 Rscript -e 'batchtools::doJobCollection("<%= uri %>")'