Merge branch 'develop' of github.com:HenrikBengtsson/future.batchtools into develop

Author: HenrikBengtsson

.github/workflows/R-CMD-check.yaml

@@ -4,32 +4,35 @@ name: R-CMD-check
 
 jobs:
   R-CMD-check:
+    if: "! contains(github.event.head_commit.message, '[ci skip]')"    
+
     runs-on: ${{ matrix.config.os }}
+
     name: ${{ matrix.config.os }} (${{ matrix.config.r }})
-    if: "! contains(github.event.head_commit.message, '[ci skip]')"
-    
+
     strategy:
       fail-fast: false
       matrix:
         config:
-        - { os: windows-latest, r: '3.5'}
-        - { os: windows-latest, r: '3.6'}
-        - { os: macOS-latest, r: '3.5'}
-        - { os: macOS-latest, r: '3.6'}
-        - { os: macOS-latest, r: 'devel'}
-        - { os: ubuntu-16.04, r: '3.2', cran: "https://demo.rstudiopm.com/all/__linux__/xenial/latest"}
-        - { os: ubuntu-16.04, r: '3.3', cran: "https://demo.rstudiopm.com/all/__linux__/xenial/latest"}
-        - { os: ubuntu-16.04, r: '3.4', cran: "https://demo.rstudiopm.com/all/__linux__/xenial/latest"}
-        - { os: ubuntu-16.04, r: '3.5', cran: "https://demo.rstudiopm.com/all/__linux__/xenial/latest"}
-        - { os: ubuntu-16.04, r: '3.6', cran: "https://demo.rstudiopm.com/all/__linux__/xenial/latest"}
+          - {os: windows-latest, r: 'devel'}
+#2020-09-23#          - {os: windows-latest, r: 'release'}
+          - {os: macOS-latest,   r: 'devel'}
+          - {os: macOS-latest,   r: 'release'}
+          - {os: ubuntu-20.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-20.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-20.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-20.04,   r: '3.5',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-20.04,   r: '3.4',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-20.04,   r: '3.3',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
 
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
-      CRAN: ${{ matrix.config.cran }}
-
+      R_FUTURE_RNG_ONMISUSE: error
+      RSPM: ${{ matrix.config.rspm }}
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
 
       - uses: r-lib/actions/setup-r@master
         with:
@@ -38,15 +41,19 @@ jobs:
       - uses: r-lib/actions/setup-pandoc@master
 
       - name: Query dependencies
-        run: Rscript -e "install.packages('remotes')" -e "saveRDS(remotes::dev_package_deps(dependencies = TRUE), 'depends.Rds', version = 2)"
+        run: |
+          install.packages('remotes')
+          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
+          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
+        shell: Rscript {0}
 
       - name: Cache R packages
         if: runner.os != 'Windows'
         uses: actions/cache@v1
         with:
           path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-r-${{ matrix.config.r }}-${{ hashFiles('depends.Rds') }}
-          restore-keys: ${{ runner.os }}-r-${{ matrix.config.r }}-
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
 
       - name: Install system dependencies
         if: runner.os == 'Linux'
@@ -59,20 +66,27 @@ jobs:
 
       - name: Install dependencies
         run: |
-          Rscript -e "library(remotes)" -e "update(readRDS('depends.Rds'))" -e "remotes::install_cran('rcmdcheck')"
-          Rscript -e "install.packages('.', type = 'source', repos = NULL)"
+          remotes::install_deps(dependencies = TRUE)
+          remotes::install_cran("rcmdcheck")
+          install.packages(".", repos=NULL, type="source")  ## needed by parallel workers
+        shell: Rscript {0}
+
+      - name: Session info
+        run: |
+          options(width = 100)
+          pkgs <- installed.packages()[, "Package"]
+          sessioninfo::session_info(pkgs, include_base = TRUE)
+        shell: Rscript {0}
 
       - name: Check
-        run: Rscript -e "rcmdcheck::rcmdcheck(args = '--no-manual', error_on = 'warning', check_dir = 'check')"
+        env:
+          _R_CHECK_CRAN_INCOMING_REMOTE_: false
+        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
+        shell: Rscript {0}
 
       - name: Upload check results
         if: failure()
         uses: actions/upload-artifact@master
         with:
           name: ${{ runner.os }}-r${{ matrix.config.r }}-results
           path: check
-
-      - name: Test coverage
-        if: matrix.config.os == 'ubuntu-16.04' && matrix.config.r == '3.6-HIDE'
-        run: |
-          Rscript  -e "install.packages('covr')" -e 'covr::codecov(token = "${{secrets.CODECOV_TOKEN}}")'

.gitignore

@@ -20,3 +20,4 @@
 revdep/data.sqlite
 revdep/checks/*
 revdep/library/*
+.github/

DESCRIPTION

@@ -1,10 +1,10 @@
 Package: future.batchtools
-Version: 0.9.0
+Version: 0.9.0-9000
 Depends:
     R (>= 3.2.0),
-    future (>= 1.14.0)
+    future (>= 1.17.0)
 Imports:
-    batchtools (>= 0.9.11),
+    batchtools (>= 0.9.13),
     utils
 Suggests:
     future.apply,

Makefile

@@ -1,6 +1,13 @@
 include .make/Makefile
 
+spelling:
+	$(R_SCRIPT) -e "spelling::spell_check_files(c('NEWS', dir('vignettes', pattern='[.]rsp', full.names=TRUE)), ignore=readLines('inst/WORDLIST', warn=FALSE))"
+ 
 future.tests/%:
 	$(R_SCRIPT) -e "future.tests::check" --args --test-plan=$*
 
 future.tests: future.tests/future.batchtools\:\:batchtools_local
+
+spelling:
+	$(R_SCRIPT) -e "spelling::spell_check_package()"
+	$(R_SCRIPT) -e "spelling::spell_check_files(c('NEWS', dir('vignettes', pattern='[.]rsp', full.names=TRUE)), ignore=readLines('inst/WORDLIST', warn=FALSE))"

NEWS

@@ -1,6 +1,19 @@
 Package: future.batchtools
 ==========================
 
+Version: 0.9.0-9000 [2020-10-13]
+
+DOCUMENTATION:
+
+ * Document option 'future.delete' and clarify option 'future.cache.path'
+   in help("future.batchtools.options").
+
+BUG FIXES:
+
+ * If run() was called twice for a BatchtoolsFuture, it would not produce
+   a FutureError but only a regular non-classed error.
+
+
 Version: 0.9.0 [2020-04-14]
 
 SIGNIFICANT CHANGES:
@@ -27,7 +40,7 @@ NEW FEATURES:
    available on all workers on a shared file system, e.g. plan(batchtools_nnn,
    registry = list(work.dir = "~")).
 
- * It is possible to control if and how batchtools should use file compressions
+ * It is possible to control if and how batchtools should use file compression
    for exported globals and results by specifying batchtools registry parameter
    'compress'.  For example, to turn off file compression, use
    plan(batchtools_nnn, registry = list(compress = FALSE)).
@@ -112,8 +125,8 @@ BUG FIXES:
  * A bug was introduced in future.batchtools 0.7.0 that could result in "Error 
    in readLog(id, reg = reg) : Log file for job with id 1 not available" when
    using one of the batchtools backends.  It occurred when the value was
-   queried.  It was observered using 'batchtools_torque' but not when using
-   'batchools_local'.  This bug was missed because the 0.7.0 release was not
+   queried.  It was observed using 'batchtools_torque' but not when using
+   'batchtools_local'.  This bug was missed because the 0.7.0 release was not
    tested on an TORQUE/PBS HPC scheduler as it should have.
 
 

R/BatchtoolsFuture-class.R

@@ -19,8 +19,9 @@
 #' @param label (optional) Label of the future (where applicable, becomes the
 #' job name for most job schedulers).
 #'
-#' @param resources (optional) A named list passed to the batchtools template
-#' (available as variable `resources`).
+#' @param resources (optional) A named list passed to the \pkg{batchtools}
+#' template (available as variable `resources`).  See Section 'Resources'
+#' in [batchtools::submitJobs()] more details.
 #'
 #' @param workers (optional) The maximum number of workers the batchtools
 #' backend may use at any time.   Interactive and "local" backends can only
@@ -48,7 +49,6 @@
 #'
 #' @export
 #' @importFrom future Future getGlobalsAndPackages
-#' @importFrom batchtools submitJobs
 #' @keywords internal
 BatchtoolsFuture <- function(expr = NULL, envir = parent.frame(),
                              substitute = TRUE,
@@ -63,7 +63,48 @@ BatchtoolsFuture <- function(expr = NULL, envir = parent.frame(),
                              ...) {
   if (substitute) expr <- substitute(expr)
 
-  if (!is.null(label)) label <- as.character(label)
+  ## Record globals
+  gp <- getGlobalsAndPackages(expr, envir = envir, globals = globals)
+
+  future <- Future(expr = gp$expr, envir = envir, substitute = FALSE,
+                   globals = gp$globals,
+                   packages = unique(c(packages, gp$packages)),
+                   label = label,
+                   ...)
+
+  future <- as_BatchtoolsFuture(future,
+                                resources = resources,
+                                workers = workers,
+                                finalize = finalize,
+                                conf.file = conf.file,
+                                cluster.functions = cluster.functions,
+                                registry = registry)
+
+  future
+}
+
+
+## Helper function to create a BatchtoolsFuture from a vanilla Future
+#' @importFrom utils file_test
+as_BatchtoolsFuture <- function(future,
+                                resources = list(),
+                                workers = NULL,
+                                finalize = getOption("future.finalize", TRUE),
+                                conf.file = findConfFile(),
+                                cluster.functions = NULL,
+                                registry = list(),
+                                ...) {
+  if (is.function(workers)) workers <- workers()
+  if (!is.null(workers)) {
+    stop_if_not(length(workers) >= 1)
+    if (is.numeric(workers)) {
+      stop_if_not(!anyNA(workers), all(workers >= 1))
+    } else {
+      stop("Argument 'workers' should be either a numeric or a function: ",
+           mode(workers))
+    }
+  }
+  future$workers <- workers
 
   if (!is.null(cluster.functions)) {
     stop_if_not(is.list(cluster.functions))
@@ -79,35 +120,13 @@ BatchtoolsFuture <- function(expr = NULL, envir = parent.frame(),
     }
   }
 
-  if (is.function(workers)) workers <- workers()
-  if (!is.null(workers)) {
-    stop_if_not(length(workers) >= 1)
-    if (is.numeric(workers)) {
-      stop_if_not(!anyNA(workers), all(workers >= 1))
-    } else {
-      stop("Argument 'workers' should be either a numeric or a function: ",
-           mode(workers))
-    }
-  }
-
   stop_if_not(is.list(registry))
   if (length(registry) > 0L) {
     stopifnot(!is.null(names(registry)), all(nzchar(names(registry))))
   }
 
   stop_if_not(is.list(resources))
 
-  ## Record globals
-  gp <- getGlobalsAndPackages(expr, envir = envir, globals = globals)
-
-  future <- Future(expr = gp$expr, envir = envir, substitute = FALSE,
-                   workers = workers, label = label,
-                   version = "1.8", .callResult = TRUE,
-                   ...)
-
-  future$globals <- gp$globals
-  future$packages <- unique(c(packages, gp$packages))
-
   ## Create batchtools registry
   reg <- temp_registry(
     label = future$label,
@@ -342,13 +361,14 @@ result.BatchtoolsFuture <- function(future, cleanup = TRUE, ...) {
 
 
 #' @importFrom future run getExpression
-#' @importFrom batchtools batchExport batchMap saveRegistry setJobNames
+#' @importFrom batchtools batchExport batchMap saveRegistry setJobNames submitJobs
 #' @export
 run.BatchtoolsFuture <- function(future, ...) {
   if (future$state != "created") {
     label <- future$label
     if (is.null(label)) label <- "<none>"
-    stop(sprintf("A future ('%s') can only be launched once.", label))
+    msg <- sprintf("A future ('%s') can only be launched once.", label)
+    stop(FutureError(msg, future = future))
   }
 
   ## Assert that the process that created the future is

R/options.R

@@ -28,15 +28,37 @@
 #'     This option controls how many lines are displayed.
 #'     (Default: `48L`)}
 #'
-#'   \item{\option{future.cache.path} / \env{R_FUTURE_CACHE_PATH}}{
+#'   \item{\option{future.cache.path} / \env{R_FUTURE_CACHE_PATH}:}{
 #'     (character string)
 #'     An absolute or relative path specifying the root folder in which
 #'     \pkg{batchtools} registry folders are stored.
-#'     (Default: `.future`)}
+#'     This folder needs to be accessible from all hosts ("workers").
+#'     Specifically, it must _not_ be a folder that is only local to the
+#'     machine such as `file.path(tempdir(), ".future"` if an job scheduler
+#'     on a HPC environment is used.
+#'     (Default: `.future` in the current working directory)}
+#'
+#'   \item{\option{future.delete}:}{(logical)
+#'     Controls whether or not the future's \pkg{batchtools} registry folder
+#'     is deleted after the future result has been collected.
+#'     If TRUE, it is always deleted.
+#'     If FALSE, it is never deleted.
+#'     If not set or NULL, the it is deleted, unless running in non-interactive
+#'     mode and the batchtools job failed or expired, which helps to
+#'     troubleshoot when running in batch mode.
+#'     (Default: NULL (not set))}
 #' }
 #'
+#' @examples
+#' # Set an R option:
+#' options(future.cache.path = "/cluster-wide/folder/.future")
+#'
+#' # Set an environment variable:
+#' Sys.setenv(R_FUTURE_RNG_ONMISUSE = "/cluster-wide/folder/.future")
+#' 
 #' @aliases
 #' future.cache.path
+#' future.delete
 #' future.batchtools.expiration.tail
 #' future.batchtools.output
 #' future.batchtools.workers

R/resources_OP.R

@@ -5,9 +5,9 @@
 #'
 #' @param fassignment The future assignment, e.g.
 #'        \code{x \%<-\% \{ expr \}}.
-#' @param tweaks A named list (or vector) of resource
-#' batchtools parameters that should be changed relative to
-#' the current strategy.
+#' @param tweaks A named list (or vector) of resource \pkg{batchtools}
+#' parameters (see Section 'Resources' in [batchtools::submitJobs()])
+#' that should be changed relative to the current strategy.
 #'
 #' @export
 #' @importFrom future plan tweak

README.md

@@ -45,10 +45,10 @@ library("future")
 library("listenv")
 ## The first level of futures should be submitted to the
 ## cluster using batchtools.  The second level of futures
-## should be using multiprocessing, where the number of
+## should be using multisession, where the number of
 ## parallel processes is automatically decided based on
 ## what the cluster grants to each compute node.
-plan(list(batchtools_torque, multiprocess))
+plan(list(batchtools_torque, multisession))
 
 ## Find all samples (one FASTQ file per sample)
 fqs <- dir(pattern = "[.]fastq$")
@@ -77,11 +77,11 @@ bams <- as.list(bams)
 ```
 Note that a user who do not have access to a cluster could use the same script processing samples sequentially and chromosomes in parallel on a single machine using:
 ```r
-plan(list(sequential, multiprocess))
+plan(list(sequential, multisession))
 ```
 or samples in parallel and chromosomes sequentially using:
 ```r
-plan(list(multiprocess, sequential))
+plan(list(multisession, sequential))
 ```
 
 For an introduction as well as full details on how to use futures,
@@ -157,10 +157,10 @@ To specify the `resources` argument at the same time as using nested future stra
 ```r
 plan(list(
   tweak(batchtools_torque, resources = list(nodes = "1:ppn=12", vmem = "5gb")),
-  multiprocess
+  multisession
 ))
 ```
-causes the first level of futures to be submitted via the TORQUE job scheduler requesting 12 cores and 5 GiB of memory per job.  The second level of futures will be evaluated using multiprocessing using the 12 cores given to each job by the scheduler.
+causes the first level of futures to be submitted via the TORQUE job scheduler requesting 12 cores and 5 GiB of memory per job.  The second level of futures will be evaluated using multisession using the 12 cores given to each job by the scheduler.
 
 A similar filename format is used for the other types of job schedulers supported.  For instance, for Slurm the template file should be named `./batchtools.slurm.tmpl` or `~/.batchtools.slurm.tmpl` in order for
 ```r