Merge branch 'master' into froll-n0

Author: jangorecki

R/frollapply.R

@@ -262,7 +262,7 @@ frollapply = function(X, N, FUN, ..., by.column=TRUE, fill=NA, align=c("right","
   }
   ## prepare templates for errors and warnings
   err.collect = gettext("frollapply calling parallel::mccollect to collect results from forked processes raised an error.\n%s")
-  warn.collect = gettext("frollapply calling parallel::mccollect to collect results from forked processes raised a warning.\n%s")
+  warn.collect = gettext("frollapply internal call to parallel::mccollect raised a warning, FUN warnings should have been suppressed by parallel.\n%s")
   if (is.function(simplify)) {
     err.simplify = gettext("frollapply completed successfully but raised an error when attempting to simplify results using user specified function in 'simplify' argument. Be sure to provide 'fill' argument matching the type and shape of results returned by the your function. Use simplify=FALSE to obtain a list instead.\n%s")
     warn.simplify = gettext("frollapply completed successfully but raised a warning when attempting to simplify results using user specified function in 'simplify' argument. Be sure to provide 'fill' argument matching the type and shape of results returned by the your function. Use simplify=FALSE to obtain a list instead.\n%s")
@@ -317,7 +317,9 @@ frollapply = function(X, N, FUN, ..., by.column=TRUE, fill=NA, align=c("right","
             tryCatch(
               parallel::mccollect(jobs),
               error = function(e) stopf(err.collect, e[["message"]]),
-              warning = function(w) warningf(warn.collect, w[["message"]])
+              warning = function(w) {
+                warningf(warn.collect, w[["message"]]) # nocov
+              }
             ),
             interrupt = function(e) {
               # nocov start
@@ -348,11 +350,14 @@ frollapply = function(X, N, FUN, ..., by.column=TRUE, fill=NA, align=c("right","
           tryCatch(
             thisans <- lapply(ansi, FUN = tightFUN, dest = cpy(w), src = thisx, n = thisn),
             error = function(e) h$err = conditionMessage(e)
-          ), warning = function(w) {h$warn = c(h$warn, conditionMessage(w)); invokeRestart("muffleWarning")}
+          ), warning = function(w) {
+            #h$warn = c(h$warn, conditionMessage(w)) ## warnings are suppressed for consistency to parallel processing code
+            invokeRestart("muffleWarning")
+          }
         )
         setDTthreads(oldDTthreads)
-        if (!is.null(h$warn))
-          warningf("frollapply received a warning(s) when evaluating FUN:\n%s", paste(unique(h$warn), collapse="\n"))
+        #if (!is.null(h$warn)) ## warnings are suppressed for consistency to parallel processing code
+        #  warningf("frollapply received a warning(s) when evaluating FUN:\n%s", paste(unique(h$warn), collapse="\n"))
         if (!is.null(h$err))
           stopf("frollapply received an error(s) when evaluating FUN:\n%s", h$err)
       }

inst/tests/froll.Rraw

@@ -1249,12 +1249,17 @@ if (use.fork) {
 }
 old = setDTthreads(1L)
 options(datatable.verbose=TRUE)
-test(6010.025, frollapply(1:2, 1, identity), c(2L,2L), output="running on single CPU thread")
+test(6010.025, frollapply(1:2, 1, copy), c(1L,2L), output="running on single CPU thread")
 options(datatable.verbose=FALSE)
-test(6010.026, frollapply(1:2, 1, function(x) {warning("warn"); x}), c(2L,2L), warning="warn")
-test(6010.027, frollapply(1:2, 1, function(x) {warning("warn:", tail(x,1)); x}), c(2L,2L), warning="warn:1\nwarn:2")
-test(6010.028, frollapply(1:2, 1, function(x) {stop("err:", tail(x,1)); x}), error="err:1") ## only first
+test(6010.026, frollapply(1:2, 1, function(x) {warning("warn"); copy(x)}), c(1L,2L))
+test(6010.027, frollapply(1:2, 1, function(x) {warning("warn:", tail(x,1)); copy(x)}), c(1L,2L))
+test(6010.028, frollapply(1:2, 1, function(x) {stop("err:", tail(x,1)); copy(x)}), error="err:1") ## only first
 setDTthreads(old)
+if (getDTthreads()>1L) { ## check for consistency
+  test(6010.036, frollapply(1:2, 1, function(x) {warning("warn"); copy(x)}), c(1L,2L))
+  test(6010.037, frollapply(1:2, 1, function(x) {warning("warn:", tail(x,1)); copy(x)}), c(1L,2L))
+  test(6010.038, frollapply(1:2, 1, function(x) {stop("err:", tail(x,1)); copy(x)}), error="err:1") ## only first
+}
 
 #### corner cases from examples - handled properly after frollapply rewrite to R
 test(6010.101, frollapply(1:5, 3, function(x) head(x, 2)), list(NA, NA, 1:2, 2:3, 3:4))

man/frollapply.Rd

@@ -74,7 +74,7 @@ frollapply(c(1, 9), N=1L, FUN=function(x) copy(list(x)))
 }
     \item \code{FUN} calls are internally passed to \code{parallel::mcparallel} to evaluate them in parallel. We inherit few limitations from \code{parallel} package explained below. This optimization can be disabled completely by calling \code{setDTthreads(1)}, then limitations listed below do not apply because all iterations of \code{FUN} evaluation will be made sequentially without use of \code{parallel} package. Note that on Windows platform this optimization is always disabled due to lack of \emph{fork} used by \code{parallel} package. One can use \code{options(datatable.verbose=TRUE)} to get extra information if \code{frollapply} is running multithreaded or not.
       \itemize{
-        \item Warnings produced inside the function are silently ignored.
+        \item Warnings produced inside the function are silently ignored; for consistency we ignore warnings also when running single threaded path.
         \item \code{FUN} should not use any on-screen devices, GUI elements, tcltk, multithreaded libraries. Note that \code{setDTthreads(1L)} is passed to forked processes, therefore any data.table code inside \code{FUN} will be forced to be single threaded. It is advised to not call \code{setDTthreads} inside \code{FUN}. \code{frollapply} is already parallelized and nested parallelism is rarely a good idea.
         \item Any operation that could misbehave when run in parallel has to be handled. For example writing to the same file from multiple CPU threads.
 \preformatted{