Merge branch 'Rdatatable:master' into master

Author: sisyphuswastaken

.Rbuildignore

@@ -17,6 +17,7 @@
 ^\.devcontainer$
 ^\.graphics$
 ^\.github$
+^\.jj$
 ^\.vscode$
 ^\.zed$
 ^\.lintr$

.github/workflows/test-coverage.yaml

@@ -51,11 +51,11 @@ jobs:
           covr::to_cobertura(cov)
         shell: Rscript {0}
 
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         with:
-          fail_ci_if_error: ${{ github.event_name != 'pull_request' && true || false }}
-          file: ./cobertura.xml
-          plugin: noop
+          fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }}
+          files: ./cobertura.xml
+          plugins: noop
           disable_search: true
           token: ${{ secrets.CODECOV_TOKEN }}
 

DESCRIPTION

@@ -3,7 +3,7 @@ Version: 1.18.99
 Title: Extension of `data.frame`
 Depends: R (>= 3.5.0)
 Imports: methods
-Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), R.utils, xts, zoo (>= 1.8-1), yaml, litedown
+Suggests: bit64 (>= 4.0.0), R.utils, xts, zoo (>= 1.8-1), yaml, litedown
 Enhances: knitr, xfun
 Description: Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.
 License: MPL-2.0 | file LICENSE

NEWS.md

@@ -30,6 +30,8 @@
 
 2. `set()` now automatically pre-allocates new column slots if needed, similar to what `:=` already does, [#1831](https://github.com/Rdatatable/data.table/issues/1831) [#4100](https://github.com/Rdatatable/data.table/issues/4100). Thanks to @zachokeeffe and @tyner for the report and @ben-schwen for the fix.
 
+3. `fread("file://...")` works for file URIs with spaces, [#7550](https://github.com/Rdatatable/data.table/issues/7550). Thanks @aitap for the report and @MichaelChirico for the PR.
+
 ## data.table [v1.18.0](https://github.com/Rdatatable/data.table/milestone/37?closed=1)  23 December 2025
 
 ### BREAKING CHANGE

R/fread.R

@@ -63,7 +63,7 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC")
       # input is data itself containing at least one \n or \r
     } else if (startsWith(input, " ")) {
       stopf("input= contains no \\n or \\r, but starts with a space. Please remove the leading space, or use text=, file= or cmd=")
-    } else if (length(grep(' ', input, fixed=TRUE)) && !file.exists(input)) {  # file name or path containing spaces is not a command
+    } else if (length(grep(' ', input, fixed=TRUE)) && !file.exists(gsub("^file://", "", input))) {  # file name or path containing spaces is not a command. file.exists() doesn't understand file:// (#7550)
       cmd = input
       if (input_has_vars && getOption("datatable.fread.input.cmd.message", TRUE)) {
         messagef("Taking input= as a system command because it contains a space ('%s'). If it's a filename please remove the space, or use file= explicitly. A variable is being passed to input= and when this is taken as a system command there is a security concern if you are creating an app, the app could have a malicious user, and the app is not running in a secure environment; e.g. the app is running as root. Please read item 5 in the NEWS file for v1.11.6 for more information and for the option to suppress this message.", cmd)

inst/tests/froll.Rraw

@@ -11,7 +11,7 @@ if (exists("test.data.table", .GlobalEnv, inherits=FALSE)) {
 
 exact_NaN = isTRUE(capabilities()["long.double"]) && identical(as.integer(.Machine$longdouble.digits), 64L)
 if (!exact_NaN) {
-  cat("\n**** Skipping 7 NaN/NA algo='exact' tests because .Machine$longdouble.digits==", .Machine$longdouble.digits, " (!=64); e.g. under valgrind\n\n", sep="")
+  cat("\n**** Skipping 8 NaN/NA algo='exact' tests because .Machine$longdouble.digits==", .Machine$longdouble.digits, " (!=64); e.g. under valgrind\n\n", sep="")
   # for Matt when he runs valgrind it is 53, but 64 when running regular R
   # froll.c uses long double and appears to require full long double accuracy in the algo='exact'
 }
@@ -1448,9 +1448,12 @@ test(6001.727, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), algo="exact"), c(NA_
 test(6001.728, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), algo="exact", na.rm=TRUE), c(NA_real_,NA_real_,NA_real_))
 test(6001.729, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), algo="exact", na.rm=TRUE, partial=TRUE), c(NA_real_,NA_real_,NA_real_))
 test(6001.730, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), fill=99, algo="exact", na.rm=TRUE), c(99,NA,NA))
-y = c(1e8+2.980232e-8, 1e8, 1e8, 1e8) # CLAMP0 test
-test(6001.731, frollvar(y, 3)[4L], 0)
-test(6001.732, frollsd(y, 3)[4L], 0)
+# numerical stability: we need to guarantee frollvar(x, n) >= 0 for all x, n
+#   the exact epsilon here is a bit implementation-dependent (as in #7546), but what's
+#   crucial is the output is never negative (or NaN after sqrt() for frollsd).
+y = c(1e8+2.980232e-8, 1e8, 1e8, 1e8)
+test(6001.731, between(frollvar(y, 3)[4L], 0, 1e-7))
+test(6001.732, between(frollsd(y, 3)[4L], 0, 1e-7))
 test(6001.733, frollvar(y, c(3,3,3,3), adaptive=TRUE)[4L], 0)
 test(6001.734, frollsd(y, c(3,3,3,3), adaptive=TRUE)[4L], 0)
 test(6001.740, frollvar(c(1.5,2.5,2,NA), c(3,3)), list(c(NA,NA,0.25,NA), c(NA,NA,0.25,NA)), output="running sequentially, because outer parallelism has been used", options=c(datatable.verbose=TRUE)) # ensure no nested parallelism in rolling functions #7352

inst/tests/knitr.Rmd

@@ -0,0 +1,10 @@
+```{r test_id, message=FALSE, results="show", echo=TRUE, warning=FALSE}
+require(data.table)              # print?
+DT = data.table(x=1:3, y=4:6)    # no
+DT                               # yes
+DT[, z := 7:9]                   # no
+print(DT[, z := 10:12])          # yes
+if (1 < 2) DT[, a := 1L]         # no
+DT                               # yes
+```
+Some text.

inst/tests/knitr.md.save

@@ -0,0 +1,41 @@
+
+``` r
+require(data.table)              # print?
+DT = data.table(x=1:3, y=4:6)    # no
+DT                               # yes
+```
+
+```
+##        x     y
+##    <int> <int>
+## 1:     1     4
+## 2:     2     5
+## 3:     3     6
+```
+
+``` r
+DT[, z := 7:9]                   # no
+print(DT[, z := 10:12])          # yes
+```
+
+```
+##        x     y     z
+##    <int> <int> <int>
+## 1:     1     4    10
+## 2:     2     5    11
+## 3:     3     6    12
+```
+
+``` r
+if (1 < 2) DT[, a := 1L]         # no
+DT                               # yes
+```
+
+```
+##        x     y     z     a
+##    <int> <int> <int> <int>
+## 1:     1     4    10     1
+## 2:     2     5    11     1
+## 3:     3     6    12     1
+```
+Some text.

inst/tests/other.Rraw

@@ -11,9 +11,12 @@ pkgs = c("DBI", "RSQLite", "bit64", "caret", "dplyr", "gdata", "ggplot2", "hexbi
 #  "yaml"            # for fread's yaml argument (csvy capability)
 #  # zoo             # In DESCRIPTION:Suggests otherwise R CMD check warning: '::' or ':::' import not declared from: 'zoo'; it is tested in other.Rraw though
 
-if (exists("test.data.table",.GlobalEnv,inherits=FALSE) ||
+if (exists("test.data.table", .GlobalEnv, inherits=FALSE) ||
     !"package:data.table" %in% search()) {
-  stop("Usage: R CMD INSTALL; require(data.table); test.data.table('other.Rraw')")
+  stop(
+    "Usage: R CMD INSTALL; require(data.table); test.data.table('other.Rraw')\n",
+    "  Did you try running it under cc()?"
+  )
   # running other.Raw in dev mode (i.e. when data.table is not installed) is not intended to work
 }
 
@@ -47,7 +50,7 @@ cat("\n")
 print(sessionInfo())
 cat("\n")
 
-if (all(c("package:reshape","package:reshape2") %in% search())) {
+if (all(c("package:reshape", "package:reshape2") %in% search())) {
   warning("Packages 'reshape' and 'reshape2' are both loaded. There have been problems before when you don't use the :: namespace prefix to disambiguate. Probably best to either remove.packages('reshape') and use reshape2 instead, or always use :: when packages mask non-generic names.")
 }
 
@@ -181,10 +184,14 @@ if (loaded[["knitr"]]) {
   # kable in knitr v1.6 uses DF[...] syntax inside it but the user might have passed a data.table.
   # Which is fine and works thanks to cedta().
   DT = data.table(x=1, y=2)
-  test(11, kable(DT), output="x.*y.*1.*2")
-  invisible(knit(testDir("knitr.Rmd"), quiet=TRUE))
-  cat(readLines("knitr.md"), sep="\n")
-  invisible(file.remove("knitr.md"))
+  test(11.1, kable(DT), output="x.*y.*1.*2")
+  local({
+    old = options(datatable.print.class=TRUE)
+    tmp = tempfile()
+    on.exit({unlink(tmp); options(old)})
+    invisible(knit(testDir("knitr.Rmd"), tmp, quiet=TRUE))
+    test(11.2, tools::Rdiff(tmp, testDir("knitr.md.save")), 0L)
+  })
 }
 
 if (loaded[["parallel"]]) {
@@ -531,6 +538,9 @@ if (loaded[["xts"]]) {  # was 1465 in tests.Rraw, #5516
 # was 2108 in tests.Rraw, #5516
 # first and last should no longer load xts namespace, #3857, below commented test for interactive validation when xts present but not loaded or attached
 # stopifnot("xts"%in%installed.packages(), !isNamespaceLoaded("xts")); library(data.table); x=as.POSIXct("2019-01-01"); last(x); stopifnot(!isNamespaceLoaded("xts"))
+# in case the search path is wrong
+if (!identical(last, data.table::last)) last = data.table::last
+if (!identical(first, data.table::first)) first = data.table::first
 local({
   x = as.POSIXct("2019-09-09")+0:1
   old = options(datatable.verbose=TRUE); on.exit(options(old))
@@ -686,6 +696,8 @@ if (loaded[["nanotime"]]) {
   # was 2080.01-05 in tests.Rraw, #5516
   n = nanotime(1:4)
   n[2L] = NA
+  # in case the search path is wrong
+  if (!identical(between, data.table::between)) between = data.table::between
   local({
     old = options(datatable.verbose=TRUE); on.exit(options(old))
     test(24.1, between(n, nanotime(2), nanotime(10)), c(FALSE, NA, TRUE, TRUE), output="between parallel processing of integer64")
@@ -779,15 +791,15 @@ if (loaded[["nanotime"]]) {
 
 # tables() with large environment #6607
 .e <- new.env() ## to not populate the .GlobalEnv
-.e[["DT"]] <- as.data.table(lapply(1:15,function(i) runif(20e6)))
-res <- tables(env=.e)
+.e[["DT"]] <- setDT(replicate(15L, runif(2e7), simplify=FALSE))
+invisible(capture.output(res <- tables(env=.e)))
 test(32, res[, .(NAME,NROW,NCOL,MB)], data.table(NAME="DT",NROW=20000000L,NCOL=15L,MB=2288.0))
 rm(.e, res)
 
 if (loaded[["vctrs"]]) {
   # vctrs::list_of() columns are treated the same as other list() columns
   DT = data.table(a = 1, b = list_of(mtcars))
-  test(33, DT, output="<vctrs_list_of>.*<data\\.frame\\[32x11\\]>")
+  test(33, options=c(datatable.print.class=TRUE), DT, output="<vctrs_list_of>.*<data\\.frame\\[32x11\\]>")
 }
 
 # NB: currently, RSQLite requires DBI, so partially redundant, but future-proof.

inst/tests/tests.Rraw

@@ -22008,3 +22008,13 @@ test(2356.2, options=c(datatable.alloccol=1L), set(DT, j="c", value=3), data.tab
 # ensure := and set are consistent if they need to overallocate
 DT = data.table(); DT2 = data.table()
 test(2356.3, options=c(datatable.alloccol=1L), {for (i in seq(10L)) set(DT, j = sprintf("V%d",i), value = i); DT}, {for (i in seq(10)) DT2[, sprintf("V%d",i) := i]; DT2})
+
+# fread works on file:// URIs with spaces, #7550
+local({
+  f = tempfile("with spaces"); on.exit(unlink(f))
+  DT = data.table(a = 1L, b = 2L)
+  fwrite(DT, f)
+
+  test(2357.1, fread(f), DT)
+  test(2357.2, fread(paste0("file://", f)), DT)
+})