Add explicit **batchtools** arguments scheduler.latency and fs.latency to batchtools backends, where applicable.

Author: HenrikBengtsson

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: future.batchtools
-Version: 0.12.2-9927
+Version: 0.12.2-9928
 Depends:
     R (>= 3.2.0),
     parallelly,

NEWS.md

@@ -21,16 +21,20 @@
  * Add `timeout <runtime> ...` to the default `batchtools_bash`
    template script to illustrate how to control this runtime via the
    backend `resources` argument.
+   
+ * Add explicit **batchtools** arguments `scheduler.latency` and
+   `fs.latency` to batchtools backends, where applicable.
 
 ## Bug Fixes
 
- * Calling `value()` on a terminated batchtools job could take 65
-   seconds to return, because it read logged output to be part of the
-   error message. When there was no output file, **batchtools** would
-   wait for the file up to 65 seconds to give job schedulers time to
-   write output to file. Now **future.batchtools** will no longer wait
-   for such files and only read their content if they exists when
-   checked.
+ * Calling `value()` on a terminated batchtools job could take a very
+   long tiem before it returned with an error. This was because it
+   read logged output to be part of the error message. When there was
+   no output file, **batchtools** would wait for the file up to
+   `fs.latency` seconds (default 65 seconds) to give job schedulers
+   and any global file system time to write output to file. Now
+   **future.batchtools** will no longer wait for such files and only
+   read their content if they exists when checked.
 
 
 # Version 0.12.2 [2025-06-06]

R/BatchtoolsFutureBackend-class.R

@@ -602,17 +602,18 @@ loggedOutput.BatchtoolsFuture <- function(future, timeout = NULL, ...) {
   if (!inherits(reg, "Registry")) return(NULL)
   jobid <- config$jobid
 
-  local({
+  out <- local({
     if (!is.null(timeout)) {
       stopifnot(length(timeout) == 1, is.numeric(timeout), !is.na(timeout), timeout >= 0.0)
       oldValue <- reg$cluster.functions$fs.latency
       on.exit(reg$cluster.functions$fs.latency <- oldValue)
       reg$cluster.functions$fs.latency <- timeout
     }
-    out <- tryCatch(suppressWarnings({
+    tryCatch(suppressWarnings({
       getLog(id = jobid, reg = reg)
     }), error = function(e) NULL)
   })
+  
   out
 } # loggedOutput()
 
@@ -822,7 +823,7 @@ await <- function(future, cleanup = TRUE, ...) {
         hint <- c("The last few lines of the logged output:", hint)
         hint <- paste(hint, collapse = "\n")
       } else {
-        hint <- "No logged output file exist"
+        hint <- "No logged output file exist (at the moment)"
       }
 
       if (any(c("submitted", "started") %in% stat)) {

R/BatchtoolsTemplateFutureBackend-class.R

@@ -5,6 +5,7 @@
 #' cluster via a job scheduler.
 #'
 #' @inheritParams BatchtoolsFutureBackend
+#' @inheritParams batchtools::makeClusterFunctions
 #'
 #' @param template (optional) A batchtools template file or a template string
 #' (in \pkg{brew} format).  If not specified, it is left to the
@@ -39,7 +40,7 @@
 #' @importFrom batchtools makeClusterFunctionsSlurm
 #' @importFrom batchtools makeClusterFunctionsTORQUE
 #' @export
-BatchtoolsTemplateFutureBackend <- function(..., template = NULL, type = c("lsf", "openlava", "sge", "slurm", "torque")) {
+BatchtoolsTemplateFutureBackend <- function(type = c("lsf", "openlava", "sge", "slurm", "torque"), scheduler.latency = 1.0, fs.latency = 65.0, template = NULL, ...) {
   assert_no_positional_args_but_first()
   type <- match.arg(type)
 
@@ -67,7 +68,7 @@ BatchtoolsTemplateFutureBackend <- function(..., template = NULL, type = c("lsf"
   template <- find_template_file(template)
 
   keep <- which(names(dotdotdot) %in% names(make_cfs_formals))
-  args <- c(list(template = template), dotdotdot[keep])
+  args <- c(list(template = template), dotdotdot[keep], scheduler.latency = scheduler.latency, fs.latency = fs.latency)
   cluster.functions <- do.call(make_cfs, args = args)
   attr(cluster.functions, "template") <- template
 

R/batchtools_bash.R

@@ -1,10 +1,13 @@
 #' @inheritParams BatchtoolsTemplateFutureBackend
+#' @inheritParams batchtools::makeClusterFunctions
+#'
 #' @rdname BatchtoolsFutureBackend
 #' @keywords internal
 #'
 #' @export
 BatchtoolsBashFutureBackend <- function(...,
-    cluster.functions = makeClusterFunctionsBash(template = template),
+    cluster.functions = makeClusterFunctionsBash(template = template, fs.latency = fs.latency),
+    fs.latency = 0.0,
     template = "bash") {
   assert_no_positional_args_but_first()
 
@@ -63,13 +66,14 @@ BatchtoolsBashFutureBackend <- function(...,
 #' 
 #' @export
 batchtools_bash <- function(
-  cluster.functions = makeClusterFunctionsBash(template = "bash"),
+  ...,
+  cluster.functions = makeClusterFunctionsBash(template = "bash", fs.latency = fs.latency),
+  fs.latency = 0.0,
   template = "bash",
   registry = list(),
   conf.file = findConfFile(),
   resources = list(),
-  finalize = getOption("future.finalize", TRUE),
-  ...) {
+  finalize = getOption("future.finalize", TRUE)) {
  stop("INTERNAL ERROR: The future.batchtools::batchtools_bash() must never be called directly")
 }
 class(batchtools_bash) <- c(
@@ -94,7 +98,7 @@ attr(batchtools_bash, "factory") <- BatchtoolsBashFutureBackend
 #' @importFrom batchtools cfReadBrewTemplate cfBrewTemplate makeClusterFunctions makeSubmitJobResult
 #' @importFrom utils file_test
 #' @export
-makeClusterFunctionsBash <- function(template = "bash") {
+makeClusterFunctionsBash <- function(template = "bash", fs.latency = 0.0) {
   bin <- Sys.which("bash")
   stop_if_not(file_test("-f", bin), file_test("-x", bin))
 
@@ -128,7 +132,8 @@ makeClusterFunctionsBash <- function(template = "bash") {
   cf <- makeClusterFunctions(
     name = "Bash",
     submitJob = submitJob,
-    store.job.collection = TRUE
+    store.job.collection = TRUE,
+    fs.latency = fs.latency
   )
   attr(cf, "template") <- template
   attr(cf, "template_text") <- template_text

R/batchtools_interactive.R

@@ -3,11 +3,11 @@
 #'
 #' @importFrom batchtools makeClusterFunctionsInteractive
 #' @export
-BatchtoolsInteractiveFutureBackend <- function(...) {
+BatchtoolsInteractiveFutureBackend <- function(fs.latency = 0.0, ...) {
   assert_no_positional_args_but_first()
 
   core <- BatchtoolsUniprocessFutureBackend(
-    cluster.functions = makeClusterFunctionsInteractive(external = FALSE),
+    cluster.functions = makeClusterFunctionsInteractive(fs.latency = fs.latency, external = FALSE),
     ...
   )
 
@@ -48,7 +48,7 @@ BatchtoolsInteractiveFutureBackend <- function(...) {
 #' @inheritParams BatchtoolsInteractiveFutureBackend
 #'
 #' @export
-batchtools_interactive <- function(...) {
+batchtools_interactive <- function(..., fs.latency = 0.0) {
  stop("INTERNAL ERROR: The future.batchtools::batchtools_interactive() must never be called directly")
 }
 class(batchtools_interactive) <- c(

R/batchtools_local.R

@@ -8,6 +8,7 @@
 #' Both types of futures will block until the futures are resolved.
 #'
 #' @inheritParams BatchtoolsFutureBackend
+#' @inheritParams batchtools::makeClusterFunctions
 #' 
 #' @param \ldots Additional arguments passed to [BatchtoolsFutureBackend()].
 #'
@@ -34,11 +35,11 @@
 #' @importFrom batchtools makeClusterFunctionsInteractive
 #' @aliases BatchtoolsLocalFutureBackend BatchtoolsBashFutureBackend
 #' @export
-BatchtoolsLocalFutureBackend <- function(...) {
+BatchtoolsLocalFutureBackend <- function(fs.latency = 0.0, ...) {
   assert_no_positional_args_but_first()
 
   core <- BatchtoolsUniprocessFutureBackend(
-    cluster.functions = makeClusterFunctionsInteractive(external = TRUE),
+    cluster.functions = makeClusterFunctionsInteractive(fs.latency = fs.latency, external = TRUE),
     ...
   )
 
@@ -77,7 +78,7 @@ BatchtoolsLocalFutureBackend <- function(...) {
 #' message("Worker process ID: ", pid)
 #' 
 #' @export
-batchtools_local <- function(...) {
+batchtools_local <- function(..., fs.latency = 0.0) {
  stop("INTERNAL ERROR: The future.batchtools::batchtools_local() must never be called directly")
 }
 class(batchtools_local) <- c(
@@ -88,6 +89,3 @@ attr(batchtools_local, "tweakable") <- c("finalize")
 attr(batchtools_local, "untweakable") <- c("workers")
 attr(batchtools_local, "init") <- TRUE
 attr(batchtools_local, "factory") <- BatchtoolsLocalFutureBackend
-
-
-

R/batchtools_lsf.R

@@ -48,7 +48,7 @@ BatchtoolsLsfFutureBackend <- function(...) {
 #' * <https://en.wikipedia.org/wiki/IBM_Spectrum_LSF>
 #'
 #' @export
-batchtools_lsf <- function(..., template = "lsf") {
+batchtools_lsf <- function(..., template = "lsf", scheduler.latency = 1.0, fs.latency = 65.0) {
  stop("INTERNAL ERROR: The future.batchtools::batchtools_lsf() must never be called directly")
 }
 class(batchtools_lsf) <- c(

R/batchtools_multicore.R

@@ -8,6 +8,7 @@
 #' multicore batchtools futures._
 #'
 #' @inheritParams BatchtoolsFutureBackend
+#' @inheritParams batchtools::makeClusterFunctions
 #'
 #' @param workers The number of multicore processes to be
 #' available for concurrent batchtools multicore futures.
@@ -31,7 +32,7 @@
 #' @importFrom parallelly availableCores supportsMulticore
 #' @importFrom tools pskill
 #' @export
-BatchtoolsMulticoreFutureBackend <- function(workers = availableCores(constraints = "multicore"), ...) {
+BatchtoolsMulticoreFutureBackend <- function(workers = availableCores(constraints = "multicore"), fs.latency = 0.0, ...) {
   assert_no_positional_args_but_first()
 
   if (is.function(workers)) workers <- workers()
@@ -46,13 +47,13 @@ BatchtoolsMulticoreFutureBackend <- function(workers = availableCores(constraint
 
   ## Fall back to batchtools_local if multicore processing is not supported
   if ((workers == 1L && !inherits(workers, "AsIs")) || !supportsMulticore(warn = TRUE)) {
-    return(BatchtoolsLocalFutureBackend(...))
+    return(BatchtoolsLocalFutureBackend(..., fs.latency = fs.latency))
   }
 
   oopts <- options(mc.cores = workers)
   on.exit(options(oopts))
 
-  cluster.functions <- makeClusterFunctionsMulticore(ncpus = workers)
+  cluster.functions <- makeClusterFunctionsMulticore(ncpus = workers, fs.latency = fs.latency)
   cluster.functions$killJob <- function(reg, batch.id) {
     pid <- as.integer(batch.id)
     if (is.na(pid) || pid <= 0) return(FALSE)
@@ -96,7 +97,7 @@ BatchtoolsMulticoreFutureBackend <- function(workers = availableCores(constraint
 #' message("Worker process ID: ", pid)
 #' 
 #' @export
-batchtools_multicore <- function(..., workers = availableCores(constraints = "multicore")) {
+batchtools_multicore <- function(..., workers = availableCores(constraints = "multicore"), fs.latency = 0.0) {
  stop("INTERNAL ERROR: The future.batchtools::batchtools_multicore() must never be called directly")
 }
 class(batchtools_multicore) <- c(

R/batchtools_openlava.R

@@ -48,7 +48,7 @@ BatchtoolsOpenLavaFutureBackend <- function(...) {
 #' * <https://en.wikipedia.org/wiki/OpenLava>
 #'
 #' @export
-batchtools_openlava <- function(..., template = "openlava") {
+batchtools_openlava <- function(..., template = "openlava", scheduler.latency = 1.0, fs.latency = 65.0) {
  stop("INTERNAL ERROR: The future.batchtools::batchtools_openlava() must never be called directly")
 }
 class(batchtools_openlava) <- c(