Merge branch 'master' into dataCodeIntegration3

Author: badasahog

.github/workflows/pkgup.yaml

@@ -12,7 +12,7 @@ on:
     branches:
       - 'master'
 
-name: pkgdown-deploy
+name: pkgup-deploy
 
 jobs:
   build:
@@ -47,16 +47,9 @@ jobs:
       - name: manual
         if: github.ref == 'refs/heads/master'
         run: |
-          cp -R ${{ env.R_LIBS_USER }} library
-          R CMD INSTALL --library="library" $(ls -1t data.table_*.tar.gz | head -n 1) --html
-          mkdir -p doc/html
-          cp $(R RHOME)/doc/html/{left.jpg,up.jpg,Rlogo.svg,R.css,index.html} doc/html
-          Rscript -e 'utils::make.packages.html("library", docdir="doc")'
-          sed -i "s|file://|../..|g" doc/html/packages.html
+          R CMD INSTALL $(ls -1t data.table_*.tar.gz | head -n 1)
           mkdir -p public
-          mv doc public/doc
-          cp -r --parents library/*/{html,help,doc,demo,DESCRIPTION,README,NEWS,README.md,NEWS.md} public 2>/dev/null || :
-          sed -i 's|"/doc/html/|"/data.table/doc/html/|g' public/library/data.table/doc/index.html 2>/dev/null || :
+          Rscript -e 'tools::pkg2HTML("data.table", out="public/index.html")'
       - name: repo
         if: github.ref == 'refs/heads/master'
         run: |

DESCRIPTION

@@ -3,7 +3,7 @@ Version: 1.17.99
 Title: Extension of `data.frame`
 Depends: R (>= 3.4.0)
 Imports: methods
-Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), R.utils, xts, zoo (>= 1.8-1), yaml, knitr, markdown
+Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), R.utils (>= 2.13.0), xts, zoo (>= 1.8-1), yaml, knitr, markdown
 Description: Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.
 License: MPL-2.0 | file LICENSE
 URL: https://r-datatable.com, https://Rdatatable.gitlab.io/data.table, https://github.com/Rdatatable/data.table

NAMESPACE

@@ -215,6 +215,7 @@ S3method(format_list_item, data.frame)
 export(fdroplevels, setdroplevels)
 S3method(droplevels, data.table)
 export(frev)
+export(.selfref.ok)
 
 # sort_by added in R 4.4.0, #6662, https://stat.ethz.ch/pipermail/r-announce/2024/000701.html
 if (getRversion() >= "4.4.0") S3method(sort_by, data.table)  

NEWS.md

@@ -311,6 +311,8 @@
 
 6. Using a double vector in `set()`'s `i=` and/or `j=` no longer throws a warning about preferring integer, [#6594](https://github.com/Rdatatable/data.table/issues/6594). While it may improve efficiency to use integer, there's no guarantee it's an improvement and the difference is likely to be minimal. The coercion will still be reported under `datatable.verbose=TRUE`. For package/production use cases, static analyzers such as `lintr::implicit_integer_linter()` can also report when numeric literals should be rewritten as integer literals.
 
+7. In rare situations a data.table object may lose its internal attribute that holds a self-reference. New helper function `.selfref.ok()` tests just that. It is only intended for technical use cases. See manual for examples.
+
 ## data.table [v1.17.8](https://github.com/Rdatatable/data.table/milestone/41) (6 July 2025)
 
 1. Internal functions used to signal errors are now marked as non-returning, silencing a compiler warning about potentially unchecked allocation failure. Thanks to Prof. Brian D. Ripley for the report and @aitap for the fix, [#7070](https://github.com/Rdatatable/data.table/pull/7070).

R/fread.R

@@ -119,6 +119,9 @@ yaml=FALSE, tmpdir=tempdir(), tz="UTC")
     if ((w <- endsWithAny(file, c(".gz", ".bgz",".bz2"))) || (gzsig <- is_gzip(file_signature)) || is_bzip(file_signature)) {
       if (!requireNamespace("R.utils", quietly = TRUE))
         stopf("To read %s files directly, fread() requires 'R.utils' package which cannot be found. Please install 'R.utils' using 'install.packages('R.utils')'.", if (w<=2L || gzsig) "gz" else "bz2") # nocov
+      # not worth doing a behavior test here, so just use getRversion().
+      if (packageVersion("R.utils") < "2.13.0" && base::getRversion() >= "4.5.0")
+        stopf("Reading compressed files in fread requires R.utils version 2.13.0 or higher. Please upgrade R.utils.") # nocov
       FUN = if (w<=2L || gzsig) gzfile else bzfile
       decompFile = tempfile(tmpdir=tmpdir)
       on.exit(unlink(decompFile), add=TRUE)

R/helpers.R

@@ -0,0 +1,19 @@
+# here we have helpers functions that DT exports, on contrary to utils.R file
+
+# convert char to factor retaining order #4837
+fctr = function(x, levels=unique(x), ..., sort=FALSE, rev=FALSE) {
+  if (!isTRUEorFALSE(sort))
+    stopf("argument 'sort' must be TRUE or FALSE")
+  if (!isTRUEorFALSE(rev))
+    stopf("argument 'rev' must be TRUE or FALSE")
+  if (sort) levels = sort(levels)
+  if (rev) levels = frev(levels)
+  factor(x, levels=levels, ...)
+}
+
+# add a function for validating data.tables that might need setDT #7329
+.selfref.ok = function(x) {
+  if (!is.data.table(x))
+    stopf(".selfref.ok expects data.table class object.")
+  selfrefok(x, verbose=FALSE) > 0L
+}

R/utils.R

@@ -220,17 +220,6 @@ rss = function() {  #5515 #5517
   # nocov end
 }
 
-# convert char to factor retaining order #4837
-fctr = function(x, levels=unique(x), ..., sort=FALSE, rev=FALSE) {
-  if (!isTRUEorFALSE(sort))
-    stopf("argument 'sort' must be TRUE or FALSE")
-  if (!isTRUEorFALSE(rev))
-    stopf("argument 'rev' must be TRUE or FALSE")
-  if (sort) levels = sort(levels)
-  if (rev) levels = frev(levels)
-  factor(x, levels=levels, ...)
-}
-
 formula_vars = function(f, x) { # .formula2varlist is not API and seems to have appeared after R-4.2, #6841
   terms <- terms(f)
   setNames(

inst/tests/froll.Rraw

@@ -1798,7 +1798,7 @@ if (use.fork) {
   setDTthreads(2)
   test(6010.763, frollapply(c(1, 9), N=1L, FUN=identity), c(1,9)) ## good only because threads >= input
   test(6010.764, frollapply(c(1, 5, 9), N=1L, FUN=identity), c(5,5,9)) ## unexpected again
-  is.ok = function(x) {stopifnot(is.data.table(x)); capture.output(print(attr(x, ".internal.selfref", TRUE)))!="<pointer: (nil)>"}
+  is.ok = function(x) {stopifnot(is.data.table(x)); out=capture.output(print(attr(x, ".internal.selfref", TRUE))); out!="<pointer: (nil)>" && out!="<pointer: 0x0>"}
   ans = frollapply(1:2, 2, data.table, simplify=FALSE) ## default: fill=NA
   test(6010.770, is.ok(ans[[2L]])) ## frollapply detected DT and fixed
   ans = frollapply(1:2, 2, data.table, fill=data.table(NA)) ## fill type match

inst/tests/tests.Rraw

@@ -147,6 +147,20 @@ if (!test_longdouble) {
 tt = Sys.getenv("TZ", unset=NA)
 TZnotUTC = !identical(tt,"") && !is_utc(tt)
 
+## nice utf8 testing code added in #7210 - to be called from inside local() !
+utf8_check_expr = quote(l10n_info()$`UTF-8` || {
+  lc_ctype = Sys.getlocale('LC_CTYPE')
+  lc_wantctype = 'en_US.UTF-8'
+  # Japanese multibyte characters require utf8. As of 2025, we're likely to be already running in a UTF-8 locale, but if not, try this setlocale() call as a last chance.
+  # Unfortunately, there is no guaranteed, portable way of switching to UTF-8 US English.
+  # Avoid the warning upon possible failure, #7210.
+  lc_newctype = suppressWarnings(Sys.setlocale('LC_CTYPE', lc_wantctype))
+  if (identical(lc_newctype, lc_wantctype)) {
+    on.exit(Sys.setlocale('LC_CTYPE', lc_ctype))
+    TRUE
+  } else FALSE
+})
+
 # generate simple error messages from base that are checked against in our tests. this helps
 #   protect us against these messages evolving in base in the future, and against these messages
 #   potentially not being produced in English.
@@ -17638,7 +17652,11 @@ test(2194.5, endsWithAny(NA_character_, 'a'), FALSE)
 test(2194.6, endsWithAny(character(), 'a'), error="Internal error.*types or lengths incorrect")
 # file used in encoding tests
 txt = readLines(testDir("issue_563_fread.txt"))
-test(2194.7, endsWithAny(txt, 'B'), error="Internal error.*types or lengths incorrect") # txt is length 5
+local(if (eval(utf8_check_expr)) {
+  test(2194.7, endsWithAny(txt, 'B'), error="Internal error.*types or lengths incorrect") # txt is length 5
+} else {
+  cat("Test 2194.7 skipped because it needs a UTF-8 locale.\n")
+})
 test(2194.8, endsWith('abcd', 'd'), error="Internal error.*use endsWithAny")
 
 # uniqueN(x, by=character()) was internal error, #4594
@@ -18619,19 +18637,8 @@ test(2252.1, dt[, b:=2L], error = "\\[ was called on a data.table.*not data.tabl
 test(2252.2, dt[, let(b=2L)], error = "\\[ was called on a data.table.*not data.table-aware.*'let'")
 rm(.datatable.aware)
 
-# tests for trunc.char handling wide characters # 5096
-local(if (l10n_info()$`UTF-8` || {
-  lc_ctype = Sys.getlocale('LC_CTYPE')
-  lc_wantctype = 'en_US.UTF-8'
-  # Japanese multibyte characters require utf8. As of 2025, we're likely to be already running in a UTF-8 locale, but if not, try this setlocale() call as a last chance.
-  # Unfortunately, there is no guaranteed, portable way of switching to UTF-8 US English.
-  # Avoid the warning upon possible failure, #7210.
-  lc_newctype = suppressWarnings(Sys.setlocale('LC_CTYPE', lc_wantctype))
-  if (identical(lc_newctype, lc_wantctype)) {
-    on.exit(Sys.setlocale('LC_CTYPE', lc_ctype))
-    TRUE
-  } else FALSE
-}) {
+# tests for trunc.char handling wide characters #5096
+local(if (eval(utf8_check_expr)) {
   accented_a = "\u0061\u0301"
   ja_ichi = "\u4E00"
   ja_ni = "\u4E8C"
@@ -20820,17 +20827,21 @@ x = data.table(a=1, b=2L)
 y = data.table(c=1.5, d=1L)
 test(2297.31, y[x, on=.(c == a, d == a), nomatch=NULL], output="Empty data.table (0 rows and 3 cols): c,d,b")
 
-# rbindlist(l, use.names=TRUE) should handle different colnames encodings #5452
-x = data.table(a = 1, b = 2, c = 3)
-y = data.table(x = 4, y = 5, z = 6)
-# a-umlaut, o-umlaut, u-umlaut
-setnames(x , c("\u00e4", "\u00f6", "\u00fc"))
-setnames(y , iconv(c("\u00f6", "\u00fc", "\u00e4"), from = "UTF-8", to = "latin1"))
-test(2298.1, rbindlist(list(x,y), use.names=TRUE), data.table("\u00e4"=c(1,6), "\u00f6"=c(2,4), "\u00fc"=c(3,5)))
-test(2298.2, rbindlist(list(y,x), use.names=TRUE), data.table("\u00f6"=c(4,2), "\u00fc"=c(5,3), "\u00e4"=c(6,1)))
-set(y, j="\u00e4", value=NULL)
-test(2298.3, rbindlist(list(x,y), use.names=TRUE, fill=TRUE), data.table("\u00e4"=c(1,NA), "\u00f6"=c(2,4), "\u00fc"=c(3,5)))
-test(2298.4, rbindlist(list(y,x), use.names=TRUE, fill=TRUE), data.table("\u00f6"=c(4,2),  "\u00fc"=c(5,3), "\u00e4"=c(NA,1)))
+local(if (eval(utf8_check_expr)) {
+  # rbindlist(l, use.names=TRUE) should handle different colnames encodings #5452
+  x = data.table(a = 1, b = 2, c = 3)
+  y = data.table(x = 4, y = 5, z = 6)
+  # a-umlaut, o-umlaut, u-umlaut
+  setnames(x , c("\u00e4", "\u00f6", "\u00fc"))
+  setnames(y , iconv(c("\u00f6", "\u00fc", "\u00e4"), from = "UTF-8", to = "latin1"))
+  test(2298.1, rbindlist(list(x,y), use.names=TRUE), data.table("\u00e4"=c(1,6), "\u00f6"=c(2,4), "\u00fc"=c(3,5)))
+  test(2298.2, rbindlist(list(y,x), use.names=TRUE), data.table("\u00f6"=c(4,2), "\u00fc"=c(5,3), "\u00e4"=c(6,1)))
+  set(y, j="\u00e4", value=NULL)
+  test(2298.3, rbindlist(list(x,y), use.names=TRUE, fill=TRUE), data.table("\u00e4"=c(1,NA), "\u00f6"=c(2,4), "\u00fc"=c(3,5)))
+  test(2298.4, rbindlist(list(y,x), use.names=TRUE, fill=TRUE), data.table("\u00f6"=c(4,2),  "\u00fc"=c(5,3), "\u00e4"=c(NA,1)))
+} else {
+  cat("Tests 2298.* skipped because they need a UTF-8 locale.\n")
+})
 
 # #6592: printing nested single-column frames
 test(2299.01, format_list_item(data.frame(a=1)),                      output="<data.frame[1x1]>")
@@ -21589,18 +21600,20 @@ test(2335.5, isoyear("2019-12-30"), 2020L)
 test(2335.6, isoyear(as.Date("2019-12-30")), 2020L)
 
 # t1-t2 for Date/IDate should be consistent, modulo storage mode #4979
-t1 = as.IDate("2025-07-01")
-t2 = as.IDate("2025-06-01")
-test(2336.1, all.equal(as.Date(t1) - as.Date(t2), t1 - t2))
-test(2336.2, all.equal(as.Date(t2) - as.Date(t1), t2 - t1))
-test(2336.3, all.equal(as.Date(t1) - t2, t1 - t2))
-test(2336.4, all.equal(as.Date(t2) - t1, t2 - t1))
-test(2336.5, all.equal(t1 - as.Date(t2), t1 - t2))
-test(2336.6, all.equal(t2 - as.Date(t1), t2 - t1))
+if (base::getRversion() >= "4.3.0") { ## follow up of #7213, see #7321
+  t1 = as.IDate("2025-07-01")
+  t2 = as.IDate("2025-06-01")
+  test(2336.1, all.equal(as.Date(t1) - as.Date(t2), t1 - t2))
+  test(2336.2, all.equal(as.Date(t2) - as.Date(t1), t2 - t1))
+  test(2336.3, all.equal(as.Date(t1) - t2, t1 - t2))
+  test(2336.4, all.equal(as.Date(t2) - t1, t2 - t1))
+  test(2336.5, all.equal(t1 - as.Date(t2), t1 - t2))
+  test(2336.6, all.equal(t2 - as.Date(t1), t2 - t1))
+}
 
 # fwrite: allow dec=',' with single column, #7227
 test(2337.1, fwrite(data.table(1), dec=","), NULL)
-if (getRversion() >= "4.0.0") { # rely on stopifnot(named = ...) for correct message
+if (base::getRversion() >= "4.0.0") { # rely on stopifnot(named = ...) for correct message
   test(2337.2, fwrite(data.table(0.1, 0.2), dec=",", sep=","), error = "dec and sep must be distinct")
 }
 test(2337.3, is.null(fwrite(data.table(c(0.1, 0.2)), dec=",", sep="\t")))
@@ -21655,3 +21668,20 @@ local({
   ...123 = 'a'
   test(2339.11, DT[, .....123], DT)
 })
+
+# Add a function for validating data.tables that might need setDT #7329
+test(2340.00, .selfref.ok(data.frame(V1=1L)), error=".selfref.ok expects data.table class object")
+d1 = structure(list(V1=1L), class=c("data.table","data.frame"))
+test(2340.01, .selfref.ok(d1), FALSE)
+setDT(d1)
+test(2340.02, .selfref.ok(d1), TRUE)
+saveRDS(d1, f<-tempfile())
+d2 = readRDS(f)
+test(2340.03, .selfref.ok(d2), FALSE)
+invisible(file.remove(f))
+setDT(d2)
+test(2340.04, .selfref.ok(d2), TRUE)
+d3 = unserialize(serialize(d2, NULL))
+test(2340.05, .selfref.ok(d3), FALSE)
+setDT(d3)
+test(2340.06, .selfref.ok(d3), TRUE)

man/.selfref.ok.Rd

@@ -0,0 +1,36 @@
+\name{.selfref.ok}
+\alias{.selfref.ok}
+\title{Tests self reference of a data.table}
+\description{
+  In rare situations, as presented in examples below, a data.table object may lose its internal attribute that holds a self-reference. This function tests just that.
+
+  It is not expected that many end users will have need for this highly technical function about data.table internals.
+}
+\usage{
+  .selfref.ok(x)
+}
+\arguments{
+  \item{x}{ A data.table. }
+}
+\value{
+  \code{TRUE} if self reference attribute is properly set, \code{FALSE} otherwise.
+}
+\examples{
+d1 = structure(list(V1=1L), class=c("data.table","data.frame"))
+.selfref.ok(d1)
+setDT(d1)
+.selfref.ok(d1)
+
+saveRDS(d1, f<-tempfile())
+d2 = readRDS(f)
+.selfref.ok(d2)
+invisible(file.remove(f))
+setDT(d2)
+.selfref.ok(d2)
+
+d3 = unserialize(serialize(d2, NULL))
+.selfref.ok(d3)
+setDT(d3)
+.selfref.ok(d3)
+}
+\keyword{ data }