NA support in frollmax #7
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,7 @@ | ||
| This is the list of follow up issues that may be resolved after all rolling functions PRs will be merged. As of the current moment none of those is a bug fix or essentially necessary but rather good practice changes. | ||
|
|
||
| - catf() instead of cat() | ||
|
|
||
| - use `test(..., options(datatable.verbose=TRUE))` rather than `options(datatable.verbose=TRUE)` | ||
|
|
||
| - verbose output `frolladaptivefun: algo 0 not implemented, fall back to 1` could be more intuitive | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -853,7 +853,7 @@ test(6000.225, frollsum(1:3, c(2,2,2), adaptive=TRUE), c(NA, 3, 5), output="frol | |
| test(6000.226, frollsum(c(NA,2,3), c(2,2,2), adaptive=TRUE), c(NA, NA, 5), output="re-running with extra care for NAs") | ||
| options(datatable.verbose=FALSE) | ||
|
|
||
| ## frollmax adaptive | ||
| options(datatable.verbose=TRUE) ## adaptive frollmax no fast algo | ||
| test(6000.3, frollmax(1:4, c(2,2,2,2), adaptive=TRUE), output="frolladaptivefun: algo 0 not implemented, fall back to 1") | ||
| test(6000.3001, frollmax(1:4, c(2,2,2,2), algo="fast", adaptive=TRUE), output="frolladaptivefun: algo 0 not implemented, fall back to 1") | ||
|
|
@@ -903,8 +903,92 @@ test(6000.3352, frollmax(x, n, hasNA=FALSE, na.rm=TRUE, adaptive=TRUE), error="d | |
| test(6000.3361, frollmax(x, n, hasNA=TRUE, adaptive=TRUE), c(NA,7,2,7,6,NA,NA,NA)) | ||
| test(6000.3362, frollmax(x, n, hasNA=TRUE, na.rm=TRUE, adaptive=TRUE), c(NA,7,2,7,6,-Inf,6,7)) | ||
|
|
||
| ## frollmax non-adaptive | ||
| options(datatable.verbose=TRUE) ## adaptive frollmax no fast algo | ||
| #test(6000.4, frollmax(1:4, c(2,2,2,2)), output="frolladaptivefun: algo 0 not implemented, fall back to 1") ## implemented | ||
| #test(6000.4001, frollmax(1:4, c(2,2,2,2), algo="fast"), output="frolladaptivefun: algo 0 not implemented, fall back to 1") | ||
| test(6000.4002, frollmax(1:4, c(2,2,2,2), algo="exact"), notOutput="frolladaptivefun: algo 0 not implemented, fall back to 1") | ||
| options(datatable.verbose=FALSE) | ||
| n = 3 | ||
| x = c(7,2,3,6,3,2,4,5) # no NA | ||
| ans = c(NA,NA,7,6,6,6,4,5) | ||
| test(6000.4111, frollmax(x, n), ans) # hasNA=NA # narm=F | ||
| test(6000.4112, frollmax(x, n, na.rm=TRUE), ans) # narm=T | ||
| test(6000.4113, frollmax(x, n, algo="exact"), ans) # hasNA=NA # narm=F | ||
| test(6000.4114, frollmax(x, n, algo="exact", na.rm=TRUE), ans) # narm=T | ||
| test(6000.4121, frollmax(x, n, hasNA=FALSE), ans) # hasNA=F | ||
| test(6000.4122, frollmax(x, n, hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4123, frollmax(x, n, algo="exact", hasNA=FALSE), ans) # hasNA=F | ||
| test(6000.4124, frollmax(x, n, algo="exact", hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4131, frollmax(x, n, hasNA=TRUE), ans) # hasNA=T | ||
| test(6000.4132, frollmax(x, n, hasNA=TRUE, na.rm=TRUE), ans) | ||
| test(6000.4133, frollmax(x, n, algo="exact", hasNA=TRUE), ans) # hasNA=T | ||
| test(6000.4134, frollmax(x, n, algo="exact", hasNA=TRUE, na.rm=TRUE), ans) | ||
| x = c(7,2,3,NA,3,2,4,NA) # NA | ||
| test(6000.4211, frollmax(x, n), c(NA,NA,7,NA,NA,NA,4,NA)) | ||
| test(6000.4212, frollmax(x, n, na.rm=TRUE), c(NA,NA,7,3,3,3,4,4)) | ||
| test(6000.4213, frollmax(x, n, algo="exact"), c(NA,NA,7,NA,NA,NA,4,NA)) | ||
| test(6000.4214, frollmax(x, n, algo="exact", na.rm=TRUE), c(NA,NA,7,3,3,3,4,4)) | ||
| test(6000.4221, frollmax(x, n, hasNA=FALSE), c(NA,NA,7,3,3,3,4,4)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4222, frollmax(x, n, hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4223, frollmax(x, n, algo="exact", hasNA=FALSE), c(NA,NA,7,3,3,3,4,4)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4224, frollmax(x, n, algo="exact", hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4231, frollmax(x, n, hasNA=TRUE), c(NA,NA,7,NA,NA,NA,4,NA)) | ||
| test(6000.4232, frollmax(x, n, hasNA=TRUE, na.rm=TRUE), c(NA,NA,7,3,3,3,4,4)) | ||
| test(6000.4233, frollmax(x, n, algo="exact", hasNA=TRUE), c(NA,NA,7,NA,NA,NA,4,NA)) | ||
| test(6000.4234, frollmax(x, n, algo="exact", hasNA=TRUE, na.rm=TRUE), c(NA,NA,7,3,3,3,4,4)) | ||
| x = rep(NA_real_, 8) # all NA | ||
| test(6000.4241, frollmax(x, n), rep(NA_real_, 8)) | ||
| test(6000.4242, frollmax(x, n, na.rm=TRUE), c(NA,NA, rep(-Inf, 6))) | ||
| test(6000.4243, frollmax(x, n, algo="exact"), rep(NA_real_, 8)) | ||
| test(6000.4244, frollmax(x, n, algo="exact", na.rm=TRUE), c(NA,NA, rep(-Inf, 6))) | ||
| test(6000.4251, frollmax(x, n, hasNA=FALSE), c(NA,NA, rep(-Inf, 6))) | ||
| test(6000.4252, frollmax(x, n, hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4253, frollmax(x, n, algo="exact", hasNA=FALSE), c(NA,NA, rep(-Inf, 6))) | ||
| test(6000.4254, frollmax(x, n, algo="exact", hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4261, frollmax(x, n, hasNA=TRUE), rep(NA_real_, 8)) | ||
| test(6000.4262, frollmax(x, n, hasNA=TRUE, na.rm=TRUE), c(NA,NA, rep(-Inf, 6))) | ||
| test(6000.4263, frollmax(x, n, algo="exact", hasNA=TRUE), rep(NA_real_, 8)) | ||
| test(6000.4264, frollmax(x, n, algo="exact", hasNA=TRUE, na.rm=TRUE), c(NA,NA, rep(-Inf, 6))) | ||
| x = c(NA,NaN,NA,NaN,NaN,NaN,NA,NA) # all NaN/NA | ||
| test(6000.4271, frollmax(x, n), c(NA,NA,NA,NA,NA,NaN,NA,NA)) | ||
| test(6000.4272, frollmax(x, n, na.rm=TRUE), c(NA,NA,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf)) | ||
| test(6000.4273, frollmax(x, n, algo="exact"), c(NA,NA,NA,NA,NA,NaN,NA,NA)) | ||
| test(6000.4274, frollmax(x, n, algo="exact", na.rm=TRUE), c(NA,NA,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf)) | ||
| test(6000.4281, frollmax(x, n, hasNA=FALSE), c(NA,NA,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4282, frollmax(x, n, hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4283, frollmax(x, n, algo="exact", hasNA=FALSE), c(NA,NA,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4284, frollmax(x, n, algo="exact", hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4291, frollmax(x, n, hasNA=TRUE), c(NA,NA,NA,NA,NA,NaN,NA,NA)) | ||
| test(6000.4292, frollmax(x, n, hasNA=TRUE, na.rm=TRUE), c(NA,NA,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf)) | ||
| test(6000.4293, frollmax(x, n, algo="exact", hasNA=TRUE), c(NA,NA,NA,NA,NA,NaN,NA,NA)) | ||
| test(6000.4294, frollmax(x, n, algo="exact", hasNA=TRUE, na.rm=TRUE), c(NA,NA,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf)) | ||
| x = c(NA,2,6,3,Inf,2,4,5) # Inf | ||
| test(6000.4311, frollmax(x, n), c(NA,NA,NA,6,Inf,Inf,Inf,5)) | ||
| test(6000.4312, frollmax(x, n, na.rm=TRUE), c(NA,NA,6,6,Inf,Inf,Inf,5)) | ||
| test(6000.4313, frollmax(x, n, algo="exact"), c(NA,NA,NA,6,Inf,Inf,Inf,5)) | ||
| test(6000.4314, frollmax(x, n, algo="exact", na.rm=TRUE), c(NA,NA,6,6,Inf,Inf,Inf,5)) | ||
| test(6000.4321, frollmax(x, n, hasNA=FALSE), c(NA,NA,6,6,Inf,Inf,Inf,5)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4322, frollmax(x, n, hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4323, frollmax(x, n, algo="exact", hasNA=FALSE), c(NA,NA,6,6,Inf,Inf,Inf,5)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4324, frollmax(x, n, algo="exact", hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4331, frollmax(x, n, hasNA=TRUE), c(NA,NA,NA,6,Inf,Inf,Inf,5)) | ||
| test(6000.4332, frollmax(x, n, hasNA=TRUE, na.rm=TRUE), c(NA,NA,6,6,Inf,Inf,Inf,5)) | ||
| test(6000.4333, frollmax(x, n, algo="exact", hasNA=TRUE), c(NA,NA,NA,6,Inf,Inf,Inf,5)) | ||
| test(6000.4334, frollmax(x, n, algo="exact", hasNA=TRUE, na.rm=TRUE), c(NA,NA,6,6,Inf,Inf,Inf,5)) | ||
| x = c(NA,2,-Inf,3,Inf,2,4,5) # -Inf | ||
| test(6000.4341, frollmax(x, n), c(NA,NA,NA,3,Inf,Inf,Inf,5)) | ||
| test(6000.4342, frollmax(x, n, na.rm=TRUE), c(NA,NA,2,3,Inf,Inf,Inf,5)) | ||
| test(6000.4343, frollmax(x, n, algo="exact"), c(NA,NA,NA,3,Inf,Inf,Inf,5)) | ||
| test(6000.4344, frollmax(x, n, algo="exact", na.rm=TRUE), c(NA,NA,2,3,Inf,Inf,Inf,5)) | ||
| test(6000.4351, frollmax(x, n, hasNA=FALSE), c(NA,NA,2,3,Inf,Inf,Inf,5)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4352, frollmax(x, n, hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4353, frollmax(x, n, algo="exact", hasNA=FALSE), c(NA,NA,2,3,Inf,Inf,Inf,5)) ## expected incorrect results, see manual hasNA section for details, added in #5441 | ||
| test(6000.4354, frollmax(x, n, algo="exact", hasNA=FALSE, na.rm=TRUE), error="does not make sense") | ||
| test(6000.4361, frollmax(x, n, hasNA=TRUE), c(NA,NA,NA,3,Inf,Inf,Inf,5)) | ||
| test(6000.4362, frollmax(x, n, hasNA=TRUE, na.rm=TRUE), c(NA,NA,2,3,Inf,Inf,Inf,5)) | ||
| test(6000.4363, frollmax(x, n, algo="exact", hasNA=TRUE), c(NA,NA,NA,3,Inf,Inf,Inf,5)) | ||
| test(6000.4364, frollmax(x, n, algo="exact", hasNA=TRUE, na.rm=TRUE), c(NA,NA,2,3,Inf,Inf,Inf,5)) | ||
|
|
||
| ## partial | ||
| x = 1:6/2 | ||
|
|
@@ -956,25 +1040,41 @@ makeNA = function(x, ratio=0.1, nf=FALSE) { | |
| } | ||
| num = 6007.0 | ||
| ## against base to verify exactness of non-finite values, not handled in zoo | ||
| rollfun = function(x, n, FUN, fill=NA_real_, na.rm=FALSE, nf.rm=FALSE, partial=FALSE) { | ||
| ans = rep(fill, nx<-length(x)) | ||
| f = match.fun(FUN) | ||
| if (nf.rm) x[is.infinite(x)] = NA_real_ | ||
| for (i in seq_along(x)) { | ||
| ans[i] = if (i >= n) | ||
| f(x[(i-n+1):i], na.rm=na.rm) | ||
| else if (partial) | ||
| f(x[max((i-n+1), 1L):i], na.rm=na.rm) | ||
| else | ||
| as.double(fill) | ||
| } | ||
| ans | ||
| } | ||
| base_compare = function(x, n, funs=c("mean","sum","max"), algos=c("fast","exact")) { | ||
| num.step = 0.001 | ||
| for (fun in funs) { | ||
| for (na.rm in c(FALSE, TRUE)) { | ||
| for (fill in c(NA_real_, 0)) { | ||
| for (partial in c(FALSE,TRUE)) { | ||
| for (algo in algos) { | ||
| num <<- num + num.step | ||
| eval(substitute( # so we can have values displayed in output/log rather than variables | ||
| test(.num, ignore.warning="no non-missing arguments", | ||
| rollfun(x, n, FUN=.fun, fill=.fill, na.rm=.na.rm, nf.rm=.nf.rm, partial=.partial), | ||
| froll(.fun, x, n, fill=.fill, na.rm=.na.rm, algo=.algo, partial=.partial)), | ||
| list(.num=num, .fun=fun, .fill=fill, .na.rm=na.rm, .algo=algo, .nf.rm=algo!="exact"&&fun!="max", .partial=partial) | ||
| )) | ||
| } | ||
| num <<- num + num.step | ||
| eval(substitute( # so we can have values displayed in output/log rather than variables | ||
| test(.num, ignore.warning="no non-missing arguments", | ||
| frollapply(x, n, FUN=match.fun(.fun), fill=.fill, na.rm=.na.rm, partial=.partial), | ||
| froll(.fun, x, n, fill=.fill, na.rm=.na.rm, algo="exact", partial=.partial)), # change to fast after inf support | ||
| list(.num=num, .fun=fun, .fill=fill, .na.rm=na.rm, .partial=partial) | ||
| )) | ||
| } | ||
| } | ||
|
|
@@ -994,20 +1094,30 @@ num = 6008.0 | |
| #### against zoo | ||
| if (requireNamespace("zoo", quietly=TRUE)) { | ||
| drollapply = function(...) as.double(zoo::rollapply(...)) # rollapply is not consistent in data type of answer, force to double | ||
| zoo_compare = function(x, n, funs=c("mean","sum","max"), algos=c("fast","exact")) { | ||
| num.step = 0.0001 | ||
| #### fun, align, na.rm, fill, algo, partial | ||
| for (fun in funs) { | ||
| for (align in c("right","center","left")) { | ||
| for (na.rm in c(FALSE, TRUE)) { | ||
| for (fill in c(NA_real_, 0)) { | ||
| for (partial in c(FALSE,TRUE)) { | ||
| if (partial && align=="center") next | ||
| for (algo in algos) { | ||
| num <<- num + num.step | ||
| eval(substitute( # so we can have values displayed in output/log rather than variables | ||
| test(.num, ignore.warning="no non-missing arguments", | ||
| drollapply(x, n, FUN=.fun, fill=.fill, align=.align, na.rm=.na.rm, partial=.partial), | ||
| froll(.fun, x, n, align=.align, fill=.fill, na.rm=.na.rm, algo=.algo, partial=.partial)), | ||
| list(.num=num, .fun=fun, .align=align, .fill=fill, .na.rm=na.rm, .algo=algo, .partial=partial) | ||
| )) | ||
| } | ||
| num <<- num + num.step | ||
| eval(substitute( # so we can have values displayed in output/log rather than variables | ||
| test(.num, ignore.warning="no non-missing arguments", | ||
| frollapply(x, n, FUN=.fun, fill=.fill, align=.align, na.rm=.na.rm, partial=.partial), | ||
| froll(.fun, x, n, align=.align, fill=.fill, na.rm=.na.rm, algo="exact", partial=.partial)), # change to fast after inf support | ||
| list(.num=num, .fun=fun, .align=align, .fill=fill, .na.rm=na.rm, .partial=partial) | ||
| )) | ||
| } | ||
| } | ||
|
|
@@ -1045,33 +1155,51 @@ if (requireNamespace("zoo", quietly=TRUE)) { | |
| } | ||
| #### adaptive moving average compare | ||
| num = 6009.0 | ||
| arollfun = function(FUN, x, n, na.rm=FALSE, align=c("right","left"), fill=NA, nf.rm=FALSE) { ## partial cannot be used with adaptive | ||
| # adaptive moving average in R | ||
| stopifnot((nx<-length(x))==length(n)) | ||
| align = match.arg(align) | ||
| ans = rep(fill, nx) | ||
| if (nf.rm) x[is.infinite(x)] = NA_real_ | ||
| f = match.fun(FUN) | ||
| if (align=="right") { | ||
| for (i in seq_along(x)) { | ||
| if (i >= n[i]) | ||
| ans[i] = f(x[(i-n[i]+1):i], na.rm=na.rm) | ||
| } | ||
| } else { | ||
| for (i in seq_along(x)) { | ||
| if (i <= nx-n[i]+1) | ||
| ans[i] = f(x[i:(i+n[i]-1)], na.rm=na.rm) | ||
| } | ||
| } | ||
| ans | ||
| } | ||
| afun_compare = function(x, n, funs=c("mean","sum","max"), algos=c("fast","exact")) { | ||
| num.step = 0.0001 | ||
| #### fun, align, na.rm, fill, algo | ||
| for (fun in funs) { | ||
| for (align in c("right","left")) { | ||
| for (na.rm in c(FALSE, TRUE)) { | ||
| for (fill in c(NA_real_, 0)) { | ||
| for (algo in algos) { | ||
| num <<- num + num.step | ||
| eval(substitute( | ||
| test(.num, ignore.warning="no non-missing arguments", | ||
| arollfun(.fun, x, n, fill=.fill, na.rm=.na.rm, nf.rm=.nf.rm, align=.align), | ||
| froll(.fun, x, n, fill=.fill, na.rm=.na.rm, algo=.algo, adaptive=TRUE, align=.align)), | ||
| list(.num=num, .fun=fun, .fill=fill, .na.rm=na.rm, .algo=algo, .nf.rm=algo!="exact"&&fun!="max", .align=align) | ||
| )) | ||
| } | ||
| if (base::getRversion() >= "3.4.0") { ## SET_GROWABLE_BIT | ||
| num <<- num + num.step | ||
| eval(substitute( | ||
| test(.num, ignore.warning="no non-missing arguments", | ||
| frollapply(x, n, FUN=match.fun(.fun), fill=.fill, na.rm=.na.rm, adaptive=TRUE, align=.align), | ||
| froll(.fun, x, n, fill=.fill, na.rm=.na.rm, algo="exact", adaptive=TRUE, align=.align)), ## change to algo=fast when mean and sum will support Inf | ||
| list(.num=num, .fun=fun, .fill=fill, .na.rm=na.rm, .nf.rm=FALSE, .align=align) | ||
| )) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -1156,8 +1284,13 @@ f = function(x) { | |
| options(datatable.verbose=FALSE) | ||
|
|
||
| # frollapply adaptive | ||
| r340 = base::getRversion() >= "3.4.0" ## support SET_GROWABLE_BIT | ||
| if (!r340) { | ||
| test(6010.2, frollapply(1:3, c(3,3,3), sum, adaptive=TRUE), error="frollapply adaptive=TRUE requires at least R 3.4.0") | ||
| } else { | ||
| test(6010.201, frollapply(1:3, c(3,2,3), sum, adaptive=TRUE), c(NA,3,6)) | ||
| test(6010.202, frollapply(1:3, c(4,4,4), sum, adaptive=TRUE), rep(NA_real_,3)) # none of the windows in k was small enough to cover length of x | ||
| } | ||
|
|
||
| #### test coverage | ||
| test(6010.501, frollapply(1:3, "b", sum), error="n must be integer") | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
let's just bump to 3.4.0 dependency for 1.18.0: Rdatatable#6840
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this will be rewritten in Rdatatable#5575 and then it will also work on < 3.4.0 but with an extra copy on each iteration.