Merge branch 'release/0.7.1'

Author: HenrikBengtsson

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: future.batchtools
-Version: 0.7.0
+Version: 0.7.1
 Depends:
     R (>= 3.2.0),
     future (>= 1.8.1)
@@ -20,7 +20,7 @@ Description: Implementation of the Future API on top of the 'batchtools' package
     in parallel out of the box, not only on your local machine or ad-hoc
     cluster of machines, but also via high-performance compute ('HPC') job
     schedulers such as 'LSF', 'OpenLava', 'Slurm', 'SGE', and 'TORQUE' / 'PBS',
-    e.g. 'y <- future_lapply(files, FUN = process)'.
+    e.g. 'y <- future.apply::future_lapply(files, FUN = process)'.
 License: LGPL (>= 2.1)
 LazyLoad: TRUE
 URL: https://github.com/HenrikBengtsson/future.batchtools

NAMESPACE

@@ -8,8 +8,8 @@ S3method(loggedOutput,BatchtoolsFuture)
 S3method(nbrOfWorkers,batchtools)
 S3method(print,BatchtoolsFuture)
 S3method(resolved,BatchtoolsFuture)
+S3method(result,BatchtoolsFuture)
 S3method(status,BatchtoolsFuture)
-S3method(value,BatchtoolsFuture)
 export("%resources%")
 export(BatchtoolsFuture)
 export(BatchtoolsFutureError)
@@ -26,7 +26,6 @@ export(finished)
 export(loggedError)
 export(loggedOutput)
 export(status)
-export(value)
 importFrom(batchtools,batchExport)
 importFrom(batchtools,batchMap)
 importFrom(batchtools,clearRegistry)
@@ -55,8 +54,8 @@ importFrom(future,getGlobalsAndPackages)
 importFrom(future,nbrOfWorkers)
 importFrom(future,plan)
 importFrom(future,resolved)
+importFrom(future,result)
 importFrom(future,tweak)
-importFrom(future,value)
 importFrom(utils,capture.output)
 importFrom(utils,file_test)
 importFrom(utils,sessionInfo)

NEWS

@@ -1,6 +1,23 @@
 Package: future.batchtools
 ==========================
 
+Version: 0.7.1 [2018-07-18]
+
+NEW FEATURES:
+
+ o The batchtools_* backends support the handling of the standard output as
+   implemented in future (>= 1.9.0).
+ 
+BUG FIXES:
+
+ o A bug was introduced in future.batchtools 0.7.0 that could result in "Error 
+   in readLog(id, reg = reg) : Log file for job with id 1 not available" when
+   using one of the batchtools backends.  It occurred when the value was
+   queried.  It was observered using 'batchtools_torque' but not when using
+   'batchools_local'.  This bug was missed because the 0.7.0 release was not
+   tested on an TORQUE/PBS HPC scheduler as it should have.
+
+
 Version: 0.7.0 [2018-05-03]
 
 NEW FEATURES:
@@ -36,7 +53,7 @@ NEW FEATURES:
 
  o The error message for expired batchtools futures now include the last few
    lines of the logged output, which sometimes includes clues on why the future
-   expired.  For instance, if a TORQUE / PBS job use more than the allocated
+   expired.  For instance, if a TORQUE/PBS job use more than the allocated
    amount of memory it might be terminated by the scheduler leaving the message
    "PBS: job killed: vmem 1234000 exceeded limit 1048576" in the output.
 

R/BatchtoolsFuture-class.R

@@ -76,7 +76,9 @@ BatchtoolsFuture <- function(expr = NULL, envir = parent.frame(),
   gp <- getGlobalsAndPackages(expr, envir = envir, globals = globals)
 
   future <- Future(expr = gp$expr, envir = envir, substitute = FALSE,
-                   workers = workers, label = label, version = "1.8", ...)
+                   workers = workers, label = label,
+                   version = "1.8", .callResult = TRUE,
+                   ...)
 
   future$globals <- gp$globals
   future$packages <- unique(c(packages, gp$packages))
@@ -112,7 +114,7 @@ BatchtoolsFuture <- function(expr = NULL, envir = parent.frame(),
 #' @export
 #' @keywords internal
 print.BatchtoolsFuture <- function(x, ...) {
-  NextMethod("print")
+  NextMethod()
 
   ## batchtools specific
   reg <- x$config$reg
@@ -152,14 +154,13 @@ loggedOutput <- function(...) UseMethod("loggedOutput")
 #'
 #' @return A character vector or a logical scalar.
 #'
-#' @aliases status finished value
+#' @aliases status finished result
 #'          loggedError loggedOutput
 #' @keywords internal
 #'
 #' @export
 #' @export status
 #' @export finished
-#' @export value
 #' @export loggedError
 #' @export loggedOutput
 #' @importFrom batchtools getStatus
@@ -192,13 +193,15 @@ status.BatchtoolsFuture <- function(future, ...) {
   status <- status[status]
   status <- sort(names(status))
   status <- setdiff(status, c("n"))
+
+  status[status == "done"] <- "finished"
 
   result <- future$result
   if (inherits(result, "FutureResult")) {
     condition <- result$condition
     if (inherits(condition, "error")) status <- c("error", status)
   }
-  
+
   status
 }
 
@@ -208,7 +211,7 @@ status.BatchtoolsFuture <- function(future, ...) {
 finished.BatchtoolsFuture <- function(future, ...) {
   status <- status(future)
   if (is_na(status)) return(NA)
-  any(c("done", "error", "expired") %in% status)
+  any(c("finished", "error", "expired") %in% status)
 }
 
 #' @export
@@ -265,25 +268,27 @@ loggedOutput.BatchtoolsFuture <- function(future, ...) {
 #' @keywords internal
 resolved.BatchtoolsFuture <- function(x, ...) {
   ## Has internal future state already been switched to be resolved
-  resolved <- NextMethod("resolved")
+  resolved <- NextMethod()
   if (resolved) return(TRUE)
 
   ## If not, checks the batchtools registry status
   resolved <- finished(x)
   if (is.na(resolved)) return(FALSE)
-
+ 
   resolved
 }
 
-#' @importFrom future value
+#' @importFrom future result
 #' @export
 #' @keywords internal
-value.BatchtoolsFuture <- function(future, signal = TRUE,
-                                   onMissing = c("default", "error"),
-                                   default = NULL, cleanup = TRUE, ...) {
+result.BatchtoolsFuture <- function(future, cleanup = TRUE, ...) {
   ## Has the value already been collected?
-  if (future$state %in% c("done", "failed", "interrupted")) {
-    return(NextMethod("value"))
+  result <- future$result
+  if (inherits(result, "FutureResult")) return(result)
+
+  ## Has the value already been collected? - take two
+  if (future$state %in% c("finished", "failed", "interrupted")) {
+    return(NextMethod())
   }
 
   if (future$state == "created") {
@@ -292,22 +297,19 @@ value.BatchtoolsFuture <- function(future, signal = TRUE,
 
   stat <- status(future)
   if (is_na(stat)) {
-    onMissing <- match.arg(onMissing)
-    if (onMissing == "default") return(default)
     label <- future$label
     if (is.null(label)) label <- "<none>"
-    stop(sprintf("The value no longer exists (or never existed) for Future ('%s') of class %s", label, paste(sQuote(class(future)), collapse = ", "))) #nolint
+    stop(sprintf("The result no longer exists (or never existed) for Future ('%s') of class %s", label, paste(sQuote(class(future)), collapse = ", "))) #nolint
   }
 
   result <- await(future, cleanup = FALSE)
   stop_if_not(inherits(result, "FutureResult"))
   future$result <- result
   future$state <- "finished"
-  if (cleanup) delete(future, ...)
-
-  NextMethod("value")
-} # value()
+  if (cleanup) delete(future)
 
+  NextMethod()
+}
 
 
 run <- function(...) UseMethod("run")
@@ -485,7 +487,7 @@ await.BatchtoolsFuture <- function(future, cleanup = TRUE,
   mdebug("- status(): %s", paste(sQuote(stat), collapse = ", "))
   mdebug("batchtools::waitForJobs() ... done")
 
-  finished <- is_na(stat) || any(c("done", "error", "expired") %in% stat)
+  finished <- is_na(stat) || any(c("finished", "error", "expired") %in% stat)
 
   ## PROTOTYPE RESULTS BELOW:
   prototype_fields <- NULL
@@ -495,11 +497,13 @@ await.BatchtoolsFuture <- function(future, cleanup = TRUE,
     mdebug("Results:")
     label <- future$label
     if (is.null(label)) label <- "<none>"
-    if ("done" %in% stat) {
+    if ("finished" %in% stat) {
       result <- loadResult(reg = reg, id = jobid)
       if (inherits(result, "FutureResult")) {
-        prototype_fields <- c(prototype_fields, "stdout")
-        result$stdout <- getLog(id = jobid, reg = reg)
+        prototype_fields <- c(prototype_fields, "batchtools_log")
+        result[["batchtools_log"]] <- try({
+          getLog(id = jobid, reg = reg)
+        }, silent = TRUE)
         if (inherits(result$condition, "error")) {
           cleanup <- FALSE
         }
@@ -508,8 +512,7 @@ await.BatchtoolsFuture <- function(future, cleanup = TRUE,
       cleanup <- FALSE
       msg <- sprintf("BatchtoolsError in %s ('%s'): %s",
                      class(future)[1], label, loggedError(future))
-      stop(BatchtoolsFutureError(msg, future = future,
-                                 output = loggedOutput(future)))
+      stop(BatchtoolsFutureError(msg, future = future))
     } else if ("expired" %in% stat) {
       cleanup <- FALSE
       msg <- sprintf("BatchtoolsExpiration: Future ('%s') expired (registry path %s).", label, reg$file.dir)
@@ -524,7 +527,7 @@ await.BatchtoolsFuture <- function(future, cleanup = TRUE,
       } else {
         msg <- sprintf("%s. No logged output exist.", msg)
       }
-      stop(BatchtoolsFutureError(msg, future = future, output = output))
+      stop(BatchtoolsFutureError(msg, future = future))
     } else if (is_na(stat)) {
       msg <- sprintf("BatchtoolsDeleted: Cannot retrieve value. Future ('%s') deleted: %s", label, reg$file.dir) #nolint
       stop(BatchtoolsFutureError(msg, future = future))
@@ -621,13 +624,9 @@ delete.BatchtoolsFuture <- function(future,
     }
   }
 
-  ## FIXME: Make sure to collect the results before deleting
+  ## Make sure to collect the results before deleting
   ## the internal batchtools registry
-  result <- future$result
-  if (is.null(result)) {
-    value(future, signal = FALSE)
-    result <- future$result
-  }
+  result <- result(future, cleanup = FALSE)
   stop_if_not(inherits(result, "FutureResult"))
 
   ## To simplify post mortem troubleshooting in non-interactive sessions,

README.md

@@ -191,14 +191,14 @@ demo("mandelbrot", package = "future", ask = FALSE)
 ## Installation
 R package future.batchtools is available on [CRAN](https://cran.r-project.org/package=future.batchtools) and can be installed in R as:
 ```r
-install.packages('future.batchtools')
+install.packages("future.batchtools")
 ```
 
 ### Pre-release version
 
 To install the pre-release version that is available in Git branch `develop` on GitHub, use:
 ```r
-source('http://callr.org/install#HenrikBengtsson/future.batchtools@develop')
+remotes::install_github("HenrikBengtsson/future.batchtools@develop")
 ```
 This will install the package from source.  
 

appveyor.yml

@@ -5,6 +5,7 @@
 #----------------------------------------------------------------
 environment:
   _R_CHECK_FORCE_SUGGESTS_: false
+  _R_CHECK_FULL_: true
 
   matrix:
   - R_VERSION: devel

cran-comments.md

@@ -1,55 +1,64 @@
-# CRAN submission future.batchtools 0.7.0
+# CRAN submission future.batchtools 0.7.1
 
-on 2018-05-03
+on 2018-07-18
 
-This submission has been validated using 'R CMD check --as-cran' on Linux, Solaris, macOS, and Windows on the r-oldrel, r-release, and r-devel versions.
-
-I've verified that this submission causes no issues for any of the 4 reverse (non-recursive) package dependencies available on CRAN.
+## Submission 1
 
 Thanks in advance
 
+## Submission 2
 
-### R CMD check --as-cran validation
+Resubmission of future.batchtools 0.7.1 where the overall R CMD check time has been decreased significantly.
 
-The package has been verified using `R CMD check --as-cran` on:
 
-* Platform x86_64-apple-darwin13.4.0 (64-bit) [Travis CI]:
-  - R version 3.3.3 (2017-03-06)
+### R CMD check --as-cran validation
 
-* Platform x86_64-apple-darwin15.6.0 (64-bit) [Travis CI]:
-#  - R version 3.5.0 (2018-04-23)  ## future 1.8.1 not available yet
+The package has been verified using `R CMD check --as-cran` on:
 
 * Platform x86_64-apple-darwin15.6.0 (64-bit) [r-hub; single-core]:
-#  - R version 3.5.0 (2018-04-23)  ## data.table not available
+  - R version 3.5.0 (2018-04-23)
 
 * Platform x86_64-unknown-linux-gnu (64-bit) [Travis CI]:
-  - R version 3.3.3 (2017-01-27)
+  - R version 3.4.4 (2017-01-27)
   - R version 3.5.0 (2017-01-27)
-  - R Under development (unstable) (2018-05-03 r74693)
+  - R Under development (unstable) (2018-06-20 r74923)
 
 * Platform x86_64-pc-linux-gnu (64-bit) [r-hub]:
   - R version 3.4.4 (2018-03-15)
-  - R Under development (unstable) (2018-04-29 r74671)
+  - R Under development (unstable) (2018-07-16 r74967)
 
 * Platform x86_64-pc-linux-gnu (64-bit):
+  - R version 3.2.0 (2015-04-16)
+  - R version 3.3.0 (2016-05-03)
+  - R version 3.4.0 (2017-04-21)
   - R version 3.5.0 (2018-04-23)
+  - R version 3.5.1 (2018-07-02)
 
-* Platform i686-pc-linux-gnu (32-bit):
-  - R version 3.4.4 (2018-03-15)
-
-* Platform i386-pc-solaris2.10 (32-bit) [r-hub]:
-  - R version 3.5.0 Patched (2018-04-30 r74674)
+* Platform i386-w64-mingw32 (32-bit) (64-bit) [Appveyor CI]:
+  - R Under development (unstable) (2018-07-16 r74967)
 
-* Platform i386-w64-mingw32 (32-bit) [Appveyor CI]:
-  - R Under development (unstable) (2018-05-02 r74682)
+* Platform x86_64-w64-mingw32/x64 (64-bit) [Appveyor CI]:
+  - R version 3.5.1 (2018-07-02)
+  - R Under development (unstable) (2018-07-16 r74967)
 
 * Platform x86_64-w64-mingw32 (64-bit) [r-hub]:
-  - R Under development (unstable) (2018-05-02 r74679)
-
-* Platform x86_64-w64-mingw32/x64 (64-bit) [Appveyor CI]:
-  - R version 3.5.0 (2018-04-23)
-  - R Under development (unstable) (2018-05-02 r74682)
+  - R Under development (unstable) (2018-07-16 r74967)
 
 * Platform x86_64-w64-mingw32/x64 (64-bit) [win-builder]:
+  - R version 3.5.1 (2018-07-02)
+  - R Under development (unstable) (2018-07-16 r74967)
+
+
+The following setups were skipped due to non-availability:
+
+* Platform x86_64-apple-darwin13.4.0 (64-bit) [Travis CI]:
+  - R version 3.4.4 (2017-01-27)
+
+* Platform x86_64-apple-darwin15.6.0 (64-bit) [Travis CI]:
   - R version 3.5.0 (2018-04-23)
-  - R Under development (unstable) (2018-04-30 r74674)
+
+* Platform i686-pc-linux-gnu (32-bit):
+  - R version 3.4.4 (2018-03-15)
+
+* Platform i686-pc-linux-gnu (32-bit):
+  - R version 3.4.4 (2018-03-15)