Merge branch 'release/1.33.2'

Author: HenrikBengtsson

.github/workflows/R-CMD-check.yaml

@@ -43,6 +43,8 @@ jobs:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       ## Test in other locale (optional)
       LANGUAGE: ${{ matrix.config.language }}
+      ## R (>= 4.4.0) Note, no trailing underscore (sic!)
+      _R_COMPARE_LANG_OBJECTS: eqonly
       ## R CMD check
       _R_CHECK_CRAN_INCOMING_: false
       _R_CHECK_LENGTH_1_CONDITION_: true

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: future
-Version: 1.33.1
+Version: 1.33.2
 Title: Unified Parallel and Distributed Processing in R for Everyone
 Imports:
     digest,
@@ -39,5 +39,6 @@ LazyLoad: TRUE
 ByteCompile: TRUE
 URL: https://future.futureverse.org, https://github.com/HenrikBengtsson/future
 BugReports: https://github.com/HenrikBengtsson/future/issues
-RoxygenNote: 7.2.3
+Encoding: UTF-8
+RoxygenNote: 7.3.1
 Roxygen: list(markdown = TRUE)

NAMESPACE

@@ -17,6 +17,7 @@ S3method(getExpression,Future)
 S3method(getExpression,MulticoreFuture)
 S3method(getExpression,MultisessionFuture)
 S3method(getExpression,UniprocessFuture)
+S3method(globals,Future)
 S3method(journal,Future)
 S3method(journal,FutureJournal)
 S3method(journal,FutureJournalCondition)
@@ -35,6 +36,7 @@ S3method(nbrOfWorkers,cluster)
 S3method(nbrOfWorkers,future)
 S3method(nbrOfWorkers,multiprocess)
 S3method(nbrOfWorkers,uniprocess)
+S3method(packages,Future)
 S3method(plot,Mandelbrot)
 S3method(print,Future)
 S3method(print,FutureCondition)

NEWS.md

@@ -1,3 +1,24 @@
+# Version 1.33.2 [2024-03-23]
+
+## Performance
+
+ * Decreased the overhead of launching futures that occurred for future
+   strategies that used a complex `workers` argument. For example,
+   `plan(cluster, workers = cl)`, where `cl` is a `cluster` object,
+   would come with an extra overhead, because the `workers` object was
+   unnecessarily transferred to the cluster nodes.
+
+## Miscellaneous
+
+ * Now `plan(multisession, workers = I(n))`, and same for `cluster`,
+   preserves the "AsIs" class attribute on the `workers` argument so
+   that it is propagated to `parallelly::makeClusterWorkers()`.
+
+## Documentation
+
+ * Clarify that packages must not change any of the `future.*` options.
+ 
+
 # Version 1.33.1 [2023-12-21]
 
 ## Bug Fixes
@@ -6,7 +27,7 @@
    circumstances call `local()` on the global search path rather than
    `base::local()` as intended.  For example, if a package that
    exports its own `local()` function was attached, then that would be
-   called instead, often leading to a hard to troubleshoot error.
+   called instead, often leading to a hard-to-troubleshoot error.
 
 
 # Version 1.33.0 [2023-07-01]

R/ClusterRegistry.R

@@ -8,7 +8,10 @@ ClusterRegistry <- local({
 
     if (is.null(workers)) {
     } else if (is.numeric(workers)) {
+      ## Preserve class attributes, especially "AsIs"
+      clazz <- class(workers)
       workers <- as.integer(workers)
+      class(workers) <- clazz
       stop_if_not(length(workers) == 1, is.finite(workers))
     } else if (is.character(workers)) {
       stop_if_not(length(workers) >= 1, !anyNA(workers))

R/Future-class.R

@@ -736,9 +736,15 @@ getExpression.Future <- local({
   tmpl_enter_plan <- bquote_compile({
     ## covr: skip=2
     .(enter)
+    
+    ## Record the original future strategy set on this worker
+    ...future.strategy.old <- future::plan("list")
+    
     ## Prevent 'future.plan' / R_FUTURE_PLAN settings from being nested
     options(future.plan = NULL)
     Sys.unsetenv("R_FUTURE_PLAN")
+
+    ## Use the next-level-down ("popped") future strategy
     future::plan(.(strategiesR), .cleanup = FALSE, .init = FALSE)
   })
 
@@ -752,7 +758,8 @@ getExpression.Future <- local({
       Sys.unsetenv("R_FUTURE_PLAN")
     else
       Sys.setenv(R_FUTURE_PLAN = .(oenv))
-    future::plan(.(strategies), .cleanup = FALSE, .init = FALSE)
+    ## Revert to the original future strategy
+    future::plan(...future.strategy.old, .cleanup = FALSE, .init = FALSE)
     ## FIXME: If we move .(exit) here, then 'R CMD check' on MS Windows
     ## complain about leftover RscriptXXXXX temporary files. /2022-07-21
     ## .(exit)
@@ -797,9 +804,12 @@ getExpression.Future <- local({
     ## Pass down the default or the remain set of future strategies?
     strategiesR <- strategies[-1]
     ##  mdebugf("Number of remaining strategies: %d", length(strategiesR))
-  
-    ## Identify packages needed by the futures
-    if (length(strategiesR) > 0L) {
+
+    ## Use default future strategy + identify packages needed by the futures
+    if (length(strategiesR) == 0L) {
+      if (debug) mdebug("Packages needed by future strategies (n = 0): <none>")
+      strategiesR <- "default"
+    } else {
       ## Identify package namespaces needed for strategies
       pkgsS <- lapply(strategiesR, FUN = environment)
       pkgsS <- lapply(pkgsS, FUN = environmentName)
@@ -808,8 +818,6 @@ getExpression.Future <- local({
       pkgsS <- intersect(pkgsS, loadedNamespaces())
       if (debug) mdebugf("Packages needed by future strategies (n = %d): %s", length(pkgsS), paste(sQuote(pkgsS), collapse = ", "))
       pkgs <- unique(c(pkgs, pkgsS))
-    } else {
-      if (debug) mdebug("Packages needed by future strategies (n = 0): <none>")
     }
 
     ## Make sure to load and attach all package needed  
@@ -822,10 +830,6 @@ getExpression.Future <- local({
       enter <- bquote_apply(tmpl_enter_packages)
     }
 
-    ## Make sure to set all nested future strategies needed
-    ## Use default future strategy?
-    if (length(strategiesR) == 0L) strategiesR <- "default"
-
     ## Pass down future strategies
     enter <- bquote_apply(tmpl_enter_plan)
     exit <- bquote_apply(tmpl_exit_plan)
@@ -847,12 +851,14 @@ getExpression.Future <- local({
 
 globals <- function(future, ...) UseMethod("globals")
 
+#' @exportS3Method
 globals.Future <- function(future, ...) {
   future[["globals"]]
 }
 
 packages <- function(future, ...) UseMethod("packages")
 
+#' @exportS3Method
 packages.Future <- function(future, ...) {
   future[["packages"]]
 }

R/globals.R

@@ -14,7 +14,7 @@
 #'
 #' @param persistent If TRUE, non-existing globals (= identified in expression but not found in memory) are always silently ignored and assumed to be existing in the evaluation environment.  If FALSE, non-existing globals are by default ignore, but may also trigger an informative error if option \option{future.globals.onMissing} in `"error"` (should only be used for troubleshooting).
 #'
-#' @param maxSize The maximum allowed total size (in bytes) of globals - for
+#' @param maxSize The maximum allowed total size (in bytes) of globals---for
 #' the purpose of preventing too large exports / transfers happening by
 #' mistake.  If the total size of the global objects are greater than this
 #' limit, an informative error message is produced. If

R/journal.R

@@ -14,34 +14,34 @@
 #' @return
 #' A data frame of class `FutureJournal` with columns:
 #'
-#'  1. `event` (character string) - type of event that took place
-#'  2. `category` (character string) - the category of the event
-#'  3. `parent` (character string) - (to be describe)
-#'  4. `start` (POSIXct) - the timestamp when the event started
-#'  5. `at` (difftime) - the time when the event started relative to
-#'     first event
-#'  6. `duration` (difftime) - the duration of the event
+#'  1.        `event` (factor)           - type of event that took place
+#'  2.     `category` (factor)           - the category of the event
+#'  3.       `parent` (character string) - (to be describe)
+#'  4.        `start` (POSIXct)          - the timestamp when the event started
+#'  5.           `at` (difftime)         - the time when the event started
+#'                                         relative to first event
+#'  6.     `duration` (difftime)         - the duration of the event
 #'  7. `future_label` (character string) - the label of the future 
-#'  8. `future_uuid` (character string) - the UUID of the future
-#'  9. `session_uuid` (character string) - the UUID of the R session
-#'     where the event took place
+#'  8.  `future_uuid` (factor)           - the UUID of the future
+#'  9. `session_uuid` (factor)           - the UUID of the R session where the
+#'                                         event took place
 #'
 #' The common events are:
 #'
-#'  * `create`   - the future was created (an `overhead`)
-#'  * `launch`   - the future was launched (an `overhead`)
+#'  *   `create` - the future was created (an `overhead`)
+#'  *   `launch` - the future was launched (an `overhead`)
 #'  * `evaluate` - the future was evaluated (an `evaluation`)
 #'  * `resolved` - the future was queried (may be occur multiple times)
 #'                 (an `overhead`)
-#'  * `gather`   - the results was retrieved (an `overhead`)
+#'  *   `gather` - the results was retrieved (an `overhead`)
 #'
 #' but others may be added by other Future classes.
 #'
-#' Common event categorys are:
+#' Common event categories are:
 #'
 #'  * `evaluation` - processing time is spent on evaluation
-#'  * `overhead`   - processing time is spent on orchestrating the future
-#'  * `waiting`    - processing time is spent on waiting to set up or
+#'  *   `overhead` - processing time is spent on orchestrating the future
+#'  *    `waiting` - processing time is spent on waiting to set up or
 #'                   querying the future
 #'
 #' but others may be added by other Future classes.
@@ -82,10 +82,10 @@ journal.Future <- function(x, ...) {
   if (!is.element("evaluate", data$event) && !is.null(x$result)) {
     stop_if_not(is.character(session_uuid))
     x <- appendToFutureJournal(x,
-      event = "evaluate",
-      category  = "evaluation",
-      start = x$result$started,
-      stop = x$result$finished
+         event = "evaluate",
+      category = "evaluation",
+         start = x$result$started,
+          stop = x$result$finished
     )
     data <- x$.journal
     stop_if_not(length(x$result$session_uuid) == 1L, is.character(x$result$session_uuid))
@@ -117,6 +117,14 @@ journal.Future <- function(x, ...) {
   levels <- c(known_levels, other_levels)
   data$event <- factor(data$event, levels = levels)
 
+  ## Coerce 'category' to a factor
+  levels <- c("evaluation", "overhead", "waiting")
+  data$category <- factor(data$category, levels = levels)
+
+  ## Coerce 'category' to a factor
+  levels <- c("evaluation", "overhead", "waiting")
+  data$category <- factor(data$category, levels = levels)
+
   ## Sort by relative start time
   if (nrow(data) > 1L) data <- data[order(data$at), ]
 

R/options.R

@@ -7,6 +7,21 @@
 #'  change in future versions of the package.  Please use with care until
 #'  further notice._
 #'
+#' @section Packages must not change future options:
+#'
+#' Just like for other R options, as a package developer you must _not_ change
+#' any of the below `future.*` options.  Only the end-user should set these.
+#' If you find yourself having to tweak one of the options, make sure to
+#' undo your changes immediately afterward.  For example, if you want to
+#' bump up the `future.globals.maxSize` limit when creating a future,
+#' use something like the following inside your function:
+#'
+#' ```r
+#' oopts <- options(future.globals.maxSize = 1.0 * 1e9)  ## 1.0 GB
+#' on.exit(options(oopts))
+#' f <- future({ expr })  ## Launch a future with large objects
+#' ```
+#'
 #' @section Settings moved to the 'parallelly' package:
 #' Several functions have been moved to the \pkg{parallelly} package:
 #'
@@ -33,7 +48,7 @@
 #'
 #'  \item{\option{future.globals.maxSize}:}{(numeric) Maximum allowed total size (in bytes) of global variables identified. Used to prevent too large exports. If set of `+Inf`, then the check for large globals is skipped. (Default: `500 * 1024 ^ 2` = 500 MiB)}
 #'
-#'   \item{\option{future.globals.onReference}: (_beta feature - may change_)}{(character string) Controls whether the identified globals should be scanned for so called _references_ (e.g. external pointers and connections) or not.  It is unlikely that another \R process ("worker") can use a global that uses a internal reference of the master \R process - we call such objects _non-exportable globals_.
+#'   \item{\option{future.globals.onReference}: (_beta feature - may change_)}{(character string) Controls whether the identified globals should be scanned for so called _references_ (e.g. external pointers and connections) or not.  It is unlikely that another \R process ("worker") can use a global that uses a internal reference of the master \R process---we call such objects _non-exportable globals_.
 #'    If this option is `"error"`, an informative error message is produced if a non-exportable global is detected.
 #'    If `"warning"`, a warning is produced, but the processing will continue; it is likely that the future will be resolved with a run-time error unless processed in the master \R process (e.g. `plan(sequential)` and `plan(multicore)`).
 #'    If `"ignore"`, no scan is performed.
@@ -67,7 +82,7 @@
 #'
 #' @section Options for controlling package startup:
 #' \describe{
-#'  \item{\option{future.startup.script}:}{(character vector or a logical) Specifies zero of more future startup scripts to be sourced when the \pkg{future} package is _attached_. It is only the first existing script that is sourced. If none of the specified files exist, nothing is sourced - there will be neither a warning nor an error.
+#'  \item{\option{future.startup.script}:}{(character vector or a logical) Specifies zero of more future startup scripts to be sourced when the \pkg{future} package is _attached_. It is only the first existing script that is sourced. If none of the specified files exist, nothing is sourced---there will be neither a warning nor an error.
 #'  If this option is not specified, environment variable \env{R_FUTURE_STARTUP_SCRIPT} is considered, where multiple scripts may be separated by either a colon (`:`) or a semicolon (`;`). If neither is set, or either is set to `TRUE`, the default is to look for a \file{.future.R} script in the current directory and then in the user's home directory.  To disable future startup scripts, set the option or the environment variable to `FALSE`.  _Importantly_, this option is _always_ set to `FALSE` if the \pkg{future} package is loaded as part of a future expression being evaluated, e.g. in a background process. In order words, they are sourced in the main \R process but not in future processes. (Default: `TRUE` in main \R process and `FALSE` in future processes / during future evaluation)}
 #'
 #'  \item{\option{future.cmdargs}:}{(character vector) Overrides \code{\link[base]{commandArgs}()} when the \pkg{future} package is _loaded_.}
@@ -113,9 +128,9 @@
 #' environment variable \env{R_FUTURE_*} _when the \pkg{future} package is
 #' loaded_. This means that those environment variables must be set before
 #' the \pkg{future} package is loaded in order to have an effect.
-#' For example, if `R_FUTURE_RNG_ONMISUSE = "ignore"`, then option
+#' For example, if `R_FUTURE_RNG_ONMISUSE="ignore"`, then option
 #' \option{future.rng.onMisuse} is set to `"ignore"` (character string).
-#' Similarly, if `R_FUTURE_GLOBALS_MAXSIZE = "50000000"`, then option
+#' Similarly, if `R_FUTURE_GLOBALS_MAXSIZE="50000000"`, then option
 #' \option{future.globals.maxSize} is set to `50000000` (numeric).
 #'
 #' @examples

R/zzz.plan.R

@@ -5,21 +5,21 @@
 #' e.g. sequentially or in parallel.
 #'
 #' @param strategy The evaluation function (or name of it) to use
-#' for resolving a future.  If NULL, then the current strategy is returned.
+#' for resolving a future. If `NULL`, then the current strategy is returned.
 #'
 #' @param \dots Additional arguments overriding the default arguments
 #' of the evaluation function.  Which additional arguments are supported
 #' depends on what evaluation function is used, e.g. several support
-#' argument `workers` but not all.  For details, see the individual
+#' argument `workers` but not all. For details, see the individual
 #' functions of which some are linked to below.
 #"
-#' @param substitute If TRUE, the `strategy` expression is
+#' @param substitute If `TRUE`, the `strategy` expression is
 #' `substitute()`:d, otherwise not.
 #'
 #' @param .call (internal) Used for recording the call to this function.
 #'
 #' @param .skip (internal) If `TRUE`, then attempts to set a strategy
-#' that is the same as what is currently in use, will skipped.
+#' that is the same as what is currently in use, will be skipped.
 #'
 #' @param .cleanup (internal) Used to stop implicitly started clusters.
 #'
@@ -81,10 +81,10 @@
 #'
 #' @section For package developers:
 #' Please refrain from modifying the future strategy inside your packages /
-#' functions, i.e. do not call `plan()` in your code.  Instead, leave
-#' the control on what backend to use to the end user.  This idea is part of
-#' the core philosophy of the future framework - as a developer you can never
-#' know what future backends the user have access to.  Moreover, by not making
+#' functions, i.e. do not call `plan()` in your code. Instead, leave
+#' the control on what backend to use to the end user. This idea is part of
+#' the core philosophy of the future framework---as a developer you can never
+#' know what future backends the user have access to. Moreover, by not making
 #' any assumptions about what backends are available, your code will also work
 #' automatically with any new backends developed after you wrote your code.
 #'
@@ -107,17 +107,17 @@
 #' others' toes._
 #'
 #' @section Using plan() in scripts and vignettes:
-#' When writing scripts or vignettes that uses futures, try to place any
-#' call to `plan()` as far up (as early on) in the code as possible.  
+#' When writing scripts or vignettes that use futures, try to place any
+#' call to `plan()` as far up (i.e. as early on) in the code as possible.  
 #' This will help users to quickly identify where the future plan is set up
 #' and allow them to modify it to their computational resources.
 #' Even better is to leave it to the user to set the `plan()` prior to
 #' `source()`:ing the script or running the vignette.
 #' If a \file{\link{.future.R}} exists in the current directory and / or in
 #' the user's home directory, it is sourced when the \pkg{future} package is
-#' _loaded_.  Because of this, the \file{.future.R} file provides a
+#' _loaded_. Because of this, the \file{.future.R} file provides a
 #' convenient place for users to set the `plan()`.
-#' This behavior can be controlled via an \R option - see
+#' This behavior can be controlled via an \R option---see
 #' \link[future:future.options]{future options} for more details.
 #'
 #' @importFrom utils capture.output