Add support for runOSCommand(..., stderr = NA), which captures 'stderr' via file and separately from 'stdout'

Author: HenrikBengtsson

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: future.batchtools
-Version: 0.20.0-9015
+Version: 0.20.0-9016
 Depends:
     R (>= 3.2.0),
     parallelly,

NEWS.md

@@ -6,13 +6,16 @@
 
 ## New Features
 
- * Add `makeClusterFunctionsSlurm2()`, which enhances
-   `batchtools::makeClusterFunctionsSlurm()` by patching the
+ * Add `makeClusterFunctionsSlurm2()`, which patches
+   `batchtools::makeClusterFunctionsSlurm()`.  Firstly, it patches the
    `listJobsQueued()` cluster function such that it falls back to
    querying Slurm's account database (`sacct`), if the future was
    _not_ found in the Slurm job queue (`squeue`), which might be the
    case when Slurm provisions a job that was just submitted to the
-   scheduler.
+   scheduler.  Secondly, it patched the `submitJob()` cluster function
+   such that the system call to `sbatch` captures stderr separately
+   from stdout, which prevents auxillary INFO messages from `sbatch`
+   to corrupt the output to be parsed.
 
 ## Bug Fixes
 

R/makeClusterFutureSlurm2.R

@@ -12,14 +12,15 @@ patchClusterFunctionsSlurm2 <- function(cf) {
   nodename <- env[["nodename"]]
   org_listJobsQueued <- env[["listJobsQueued"]]
 
-  ## Patch submitJob() to use runOSCommand(..., stderr = FALSE)
+  ## Patch submitJob() to use runOSCommand(..., stderr = NA),
+  ## which captures 'stderr' separately from 'stdout'
   ## See https://github.com/mlr-org/batchtools/pull/314
   submitJob <- cf[["submitJob"]]
   env_submitJob <- new.env(parent = environment(submitJob))
-  env_submitJob[["runOSCommand"]] <- function(..., stderr = FALSE) {
+  env_submitJob[["runOSCommand"]] <- function(..., stderr = NA) {
     debug <- isTRUE(getOption("future.batchtools.debug"))
     if (debug) {
-      mdebugf_push("runOSCommand(..., stderr = FALSE) ...")
+      mdebugf_push("runOSCommand(..., stderr = %s) ...", stderr)
       mprint(list(args = list(..., stderr = stderr)))
       on.exit(mdebugf_pop())
     }

R/runOSCommand.R

@@ -19,16 +19,30 @@ runOSCommand = function(sys.cmd, sys.args = character(0L), stdin = "", stdout =
   "!DEBUG [runOSCommand]: cmd: `sys.cmd` `stri_flatten(sys.args, ' ')`"
 
   if (nzchar(Sys.which(sys.cmd))) {
-    res = suppressWarnings(system2(command = sys.cmd, args = sys.args, stdin = stdin, stdout = stdout, stderr = stderr, wait = TRUE))
+    ## Capture stderr separately
+    if (is.na(stderr)) {
+      stderr_file = tempfile()
+      on.exit(file.remove(stderr_file))
+    } else {
+      stderr_file = stderr
+    }
+    res = suppressWarnings(system2(command = sys.cmd, args = sys.args, stdin = stdin, stdout = stdout, stderr = stderr_file, wait = TRUE))
     output = as.character(res)
     exit.code = attr(res, "status") %??% 0L
+    if (is.na(stderr)) {
+      output_stderr = readLines(stderr_file, warn = FALSE)
+    } else {
+      output_stderr = NULL
+    }
   } else {
     output = "command not found"
+    output_stderr = NULL
     exit.code = 127L
   }
 
   "!DEBUG [runOSCommand]: OS result (stdin '`stdin`', exit code `exit.code`):"
   "!DEBUG [runOSCommand]: `paste0(output, sep = '\n')`"
+  "!DEBUG [runOSCommand]: `paste0(output_stderr, sep = '\n')`"
 
-  return(list(sys.cmd = sys.cmd, sys.args = sys.args, exit.code = exit.code, output = output))
+  return(list(sys.cmd = sys.cmd, sys.args = sys.args, exit.code = exit.code, output = output, stderr = output_stderr))
 }