Merge branch 'master' into froll2025

Author: jangorecki

.ci/README.md

@@ -13,7 +13,8 @@ Test jobs:
 - `test-lin-dev-gcc-strict-cran` - `--as-cran` on Linux, `r-devel` built with `-enable-strict-barrier --disable-long-double`, test for compilation warnings, test for new NOTEs/WARNINGs from `R CMD check`.
 - `test-lin-dev-clang-cran` - same as `gcc-strict` job but R built with `clang` and  no `--enable-strict-barrier --disable-long-double` flags.
 - `test-lin-ancient-cran` - Stated R dependency version (currently 3.4.0) on Linux.
-- `test-lin-dev-san` - `r-devel` on Linux built with `clang -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
+- `test-lin-dev-clang-san` - `r-devel` on Linux built with `clang -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
+- `test-lin-dev-gcc-san` - `r-devel` on Linux built with `gcc -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
 - `test-win-rel` - `r-release` on Windows.
 - `test-win-dev` - `r-devel` on Windows.
 - `test-win-old` - `r-oldrel` on Windows.

.gitlab-ci.yml

@@ -13,14 +13,14 @@ variables:
   TZ: "UTC"  ## to avoid 'Failed to create bus connection' from timedatectl via Sys.timezone() on Docker with R 3.4.
              ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
              ## a non-UTC timezone, although, that's what we do routinely in dev.
-  R_REL_VERSION:  "4.4" # only raise when RTOOLS for REL is available
-  R_REL_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.4.1/R-4.4.1-win.exe"
-  R_DEV_VERSION:  "4.5"
+  R_REL_VERSION:  "4.5" # only raise when RTOOLS for REL is available
+  R_REL_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.5.0/R-4.5.0-win.exe"
+  R_DEV_VERSION:  "4.6"
   R_DEV_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/R-devel-win.exe"
-  R_OLD_VERSION:  "4.3"
-  R_OLD_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.3.3/R-4.3.3-win.exe"  
-  R_REL_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.4.1-arm64.pkg"
-  R_OLD_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.3.3-arm64.pkg"
+  R_OLD_VERSION:  "4.4"
+  R_OLD_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.4.3/R-4.4.3-win.exe"
+  R_REL_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.5.0-arm64.pkg"
+  R_OLD_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.4.3-arm64.pkg"
 
 stages:
   - dependencies
@@ -48,6 +48,9 @@ mirror-packages:
   cache:
     paths:
       - bus/$CI_JOB_NAME/cran
+  variables:
+    # TODO(R-ancient>=3.5.0): remove this; let it save PACKAGES.rds in version-3 format
+    R_DEFAULT_SERIALIZE_VERSION: 2
   script:
     - echo 'source(".ci/ci.R")' >> .Rprofile
     - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib
@@ -104,21 +107,23 @@ build:
 # force all suggests
 # flags: gcc -O3 -flto=auto -fno-common -Wunused-result
 # tests for compilation warnings
+# runs the --as-cran check, including the URL checks
 test-lin-rel:
   <<: *test-lin
   image: registry.gitlab.com/rdatatable/dockerfiles/r-data.table
   variables:
-    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
-    _R_CHECK_CRAN_INCOMING_: "FALSE"
-    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
     OPENBLAS_MAIN_FREE: "1"
   script:
     - *install-deps
     - echo 'CFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
     - echo 'CXXFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
-    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
-    - (! grep "warning:" data.table.Rcheck/00install.out)
+    - echo '_R_CHECK_COMPILATION_FLAGS_KNOWN_=-Wvla' >> ~/.Renviron
+    - |
+        res1=0; R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1) || res1=$?
+        res2=0; grep -v "warning:" data.table.Rcheck/00install.out || res2=$?
+        res3=0; Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 2 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 2 NOTEs"), " (CRAN incoming feasibility, non-API) but ", shQuote(l))' || res3=$?
+        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ] || [ $res3 -ne 0 ]; then exit 1; fi
 
 ## vanilla minimal
 # no zlib
@@ -151,9 +156,10 @@ test-lin-rel-cran:
     - *install-deps
     - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
     - echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
-    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
-    - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
+    - |
+        res1=0; R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1) || res1=$?
+        res2=0; Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 1 NOTE")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 1 NOTE"), " (non-API) but ", shQuote(l))' || res2=$?
+        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ]; then exit 1; fi
 
 ## R-devel on Linux gcc strict
 # R built with --enable-strict-barrier --disable-long-double
@@ -175,7 +181,7 @@ test-lin-dev-gcc-strict-cran:
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - (! grep "warning:" data.table.Rcheck/00install.out)
     - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, installed package size, non-API calls) but ", shQuote(l)) else q("no")'
+        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls, V8 package) but ", shQuote(l)) else q("no")'
 
 ## R-devel on Linux clang
 # R compiled with clang, flags removed: -flto=auto -fopenmp
@@ -198,7 +204,7 @@ test-lin-dev-clang-cran:
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - (! grep "warning:" data.table.Rcheck/00install.out)
     - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 2 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls) but ", shQuote(l)) else q("no")'
+        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls, V8 package) but ", shQuote(l)) else q("no")'
 
 # stated dependency on R
 test-lin-ancient-cran:
@@ -212,8 +218,8 @@ test-lin-ancient-cran:
     # Restore checking vignettes if upgrading our R dependency means knitr can be installed, or when we switch to litedown.
     - R CMD check --no-manual --no-build-vignettes --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
 
-# run the main checks with Address(+Leak),UBSanitizer enabled
-test-lin-dev-san:
+# run the main checks with Address(+Leak),UBSanitizer enabled, GCC _and_ Clang
+test-lin-dev-clang-san:
   <<: *test-lin
   image: registry.gitlab.com/rdatatable/dockerfiles/r-devel-clang-san
   variables:
@@ -227,8 +233,27 @@ test-lin-dev-san:
     - |
         res1=0; ASAN_OPTIONS=detect_leaks=1 R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) || res1=$?
         res2=0; perl -nle '(print, $a=1) if /: runtime error: |ERROR: LeakSanitizer/../SUMMARY.*Sanitizer/ }{ exit $a' data.table.Rcheck/**/*.Rout* || res2=$?
+        res3=0; tail -n 1 data.table.Rcheck/00check.log | grep -q -e '^Status: [0-9]* NOTEs*$' -e '^Status: OK$' || res3=$?
         # fail if R CMD check had failed or if sanitizer output found
-        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ]; then exit 1; fi
+        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ] || [ $res3 -ne 0 ]; then exit 1; fi
+
+test-lin-dev-gcc-san:
+  <<: *test-lin
+  image: registry.gitlab.com/rdatatable/dockerfiles/r-devel-gcc-san
+  variables:
+    # must be set for most of the process because there are pseudo-leaks everywhere
+    ASAN_OPTIONS: "detect_leaks=0"
+    # fontconfig is known to leak; add more suppressions as discovered
+    LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/.dev/lsan.supp"
+    UBSAN_OPTIONS: "print_stacktrace=1"
+  script:
+    - *install-deps
+    - |
+        res1=0; ASAN_OPTIONS=detect_leaks=1 R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) || res1=$?
+        res2=0; perl -nle '(print, $a=1) if /: runtime error: |ERROR: LeakSanitizer/../SUMMARY.*Sanitizer/ }{ exit $a' data.table.Rcheck/**/*.Rout* || res2=$?
+        res3=0; tail -n 1 data.table.Rcheck/00check.log | grep -q -e '^Status: [0-9]* NOTEs*$' -e '^Status: OK$' || res3=$?
+        # fail if R CMD check had failed or if sanitizer output found
+        if [ $res1 -ne 0 ] || [ $res2 -ne 0 ] || [ $res3 -ne 0 ]; then exit 1; fi
 
 .test-win-template: &test-win
   <<: *test
@@ -290,11 +315,14 @@ test-win-old:
   before_script:
     - curl -O $R_BIN
     - sudo installer -pkg "$(ls -1t R-*-arm64.pkg | head -n 1)" -target /
+    - sudo Rscript -e "source('https://mac.R-project.org/bin/install.R'); install.libs('gettext')"
     - *install-deps
     - cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .
   script:
     - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
     - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
+    - >-
+        tail -n 1 data.table.Rcheck/00check.log | grep -q -e '^Status: [0-9]* NOTEs*$' -e '^Status: OK$'
   after_script:
     - mkdir -p bus/$CI_JOB_NAME
     - '[ -d data.table.Rcheck ] && mv data.table.Rcheck bus/$CI_JOB_NAME/'
@@ -329,7 +357,7 @@ integration:
     - saas-linux-medium-amd64
   only:
     - master
-  needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-ancient-cran","test-lin-dev-san","test-win-rel","test-win-dev" ,"test-win-old","test-mac-rel","test-mac-old"]
+  needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-ancient-cran","test-lin-dev-clang-san","test-lin-dev-gcc-san","test-win-rel","test-win-dev" ,"test-win-old","test-mac-rel","test-mac-old"]
   script:
     - R --version
     - *install-deps ## markdown pkg not present in r-pkgdown image

DESCRIPTION

@@ -55,7 +55,7 @@ Authors@R: c(
   person("James","Sams",           role="ctb"),
   person("Martin","Morgan",        role="ctb"),
   person("Michael","Quinn",        role="ctb"),
-  person("@javrucebo","",          role="ctb"),
+  person(given="@javrucebo",       role="ctb", comment="GitHub user"),
   person("Marc","Halperin",        role="ctb"),
   person("Roy","Storey",           role="ctb"),
   person("Manish","Saraswat",      role="ctb"),
@@ -102,5 +102,6 @@ Authors@R: c(
   person("Aljaž", "Sluga",         role="ctb"),
   person("Bill", "Evans",          role="ctb"),
   person("Reino", "Bruner",        role="ctb"),
-  person(comment=c(github="@badasahog"), role="ctb")
+  person(given="@badasahog",       role="ctb", comment="GitHub user"),
+  person("Vinit", "Thakur",        role="ctb")
   )

NEWS.md

@@ -91,6 +91,12 @@ As of now, adaptive rolling max has no _on-line_ implemention (`algo="fast"`), i
 
 9. Joins to extended data.frames, e.g. `x[i, col := x.col1 + i.col2]` where `i` is a `tbl`, can use the `x.` and `i.` prefix forms, [#6998](https://github.com/Rdatatable/data.table/issues/6998). Thanks @MichaelChirico for the bug and PR.
 
+10. On a heavily loaded machine, a `forder` thread could try to perform a zero-length copy from a null pointer, which was de-facto harmless but is against the C standard and was caught by additional CRAN checks, [#7051](https://github.com/Rdatatable/data.table/issues/7051). Thanks to @helske for the report and @aitap for the PR.
+
+11. Out of sample type bumps now respect `integer64=` selection, [#7032](https://github.com/Rdatatable/data.table/pull/7032).
+
+12. Internal functions used to signal errors are now marked as non-returning, silencing a compiler warning about potentially unchecked allocation failure. Thanks to Prof. Brian D. Ripley for the report and @aitap for the fix, [#7070](https://github.com/Rdatatable/data.table/pull/7070).
+
 ### NOTES
 
 1. Continued work to remove non-API C functions, [#6180](https://github.com/Rdatatable/data.table/issues/6180). Thanks Ivan Krylov for the PRs and for writing a clear and concise guide about the R API: https://aitap.codeberg.page/R-api/.

README.md

@@ -19,12 +19,12 @@
 
 The `data.table` project uses a [custom governance agreement](./GOVERNANCE.md) 
 and is fiscally sponsored by [NumFOCUS](https://numfocus.org/). Consider making 
-a [tax-deductible donation](https://numfocus.org/donate-to-data-table) to help the project 
+a [tax-deductible donation](https://numfocus.org/project/data-table) to help the project 
 pay for developer time, professional services, travel, workshops, and a variety of other needs.
 
 <div align="center">
   <a href="https://numfocus.org/project/data-table">
-    <img height="60px" 
+    <img width="25%" 
          src="https://raw.githubusercontent.com/numfocus/templates/master/images/numfocus-logo.png" 
          align="center">
   </a>

inst/tests/tests.Rraw

@@ -3070,6 +3070,16 @@ if (test_bit64) test(1017.1, fread(f), copy(DT)[,A:=as.integer64(A)])
 test(1017.2, fread(f, integer64="character"), DT)
 unlink(f)
 
+DT = data.table(a=seq(10000), b="100")
+DT[111, b := "1000000000000"]
+f = tempfile()
+fwrite(DT, f)
+
+test(1017.3, fread(f, integer64="numeric"), fread(f, colClasses=c("integer", "numeric")))
+test(1017.4, fread(f, integer64="character"), fread(f, colClasses=c("integer", "character")))
+
+unlink(f)
+
 # ERANGE errno handled, #106 #4165
 test(1018.1, identical(fread("1.46761e-313\n"), data.table(V1=1.46761e-313)))
 test(1018.2, identical(fread("1.46761e+313\n"), data.table(V1=1.46761e+313)))

man/IDateTime.Rd

@@ -137,6 +137,11 @@ hours. Because \code{ITime} is stored in seconds, you can add it to a
 \code{POSIXct} object, but you should not add it to a \code{Date}
 object.
 
+For \code{as.ITime}, note that the string \code{"24:00:00"} is parsed as \code{"00:00:00"}.
+This is because the conversion uses \code{as.POSIXct}, which treats \code{"24:00:00"} as midnight of the next day. 
+This differs from ISO 8601 (which allows \code{"24:00:00"} to represent end-of-day), but aligns with POSIX standards.
+To represent end-of-day intervals, use \code{"23:59:59"} or arithmetic (e.g., \code{as.ITime("23:59:59") + 1L}).
+
 We also provide S3 methods to convert to and from \code{Date} and \code{POSIXct}.
 
 \code{ITime} is time zone-agnostic. When converting \code{ITime} and
@@ -245,6 +250,12 @@ identical(as.ITime("10:45"), methods::as("10:45", "ITime"))
 
 (t <- as.ITime("10:45:04", format = "\%H:\%M:\%S"))
 
+# "24:00:00" is parsed as "00:00:00"
+as.ITime("24:00:00")
+
+# Workaround for end-of-day: add 1 second to "23:59:59"
+as.ITime("23:59:59") + 1L
+
 as.POSIXct("2001-01-01") + as.ITime("10:45")
 
 datetime <- seq(as.POSIXct("2001-01-01"), as.POSIXct("2001-01-03"), by = "5 hour")

man/data.table.Rd

@@ -174,7 +174,7 @@ data.table(\dots, keep.rownames=FALSE, check.names=FALSE, key=NULL, stringsAsFac
             \item For convenience during interactive scenarios, it is also possible to use \code{.()} syntax as \code{X[Y, on=.(a, b)]}.
             \item From v1.9.8, (non-equi) joins using binary operators \code{>=, >, <=, <} are also possible, e.g., \code{X[Y, on=c("x>=a", "y<=b")]}, or for interactive use as \code{X[Y, on=.(x>=a, y<=b)]}.
         }
-        See examples as well as \href{../doc/datatable-secondary-indices-and-auto-indexing.html}{\code{vignette("datatable-secondary-indices-and-auto-indexing")}}.
+        Note that providing \code{on} is \emph{required} for \code{X[Y]} joins when \code{X} is unkeyed. See examples as well as \href{../doc/datatable-secondary-indices-and-auto-indexing.html}{\code{vignette("datatable-secondary-indices-and-auto-indexing")}}.
     }
 
   \item{env}{ List or an environment, passed to \code{\link{substitute2}} for substitution of parameters in \code{i}, \code{j} and \code{by} (or \code{keyby}). Use \code{verbose} to preview constructed expressions. For more details see \href{../doc/datatable-programming.html}{\code{vignette("datatable-programming")}}. }
@@ -298,7 +298,9 @@ DT[, sum(v), by=x][order(x)]   # same but by chaining expressions together
 
 # fast ad hoc row subsets (subsets as joins)
 DT["a", on="x"]                # same as x == "a" but uses binary search (fast)
+                               #   NB: requires DT to be keyed!
 DT["a", on=.(x)]               # same, for convenience, no need to quote every column
+                               #   NB: works regardless of whether or not DT is keyed!
 DT[.("a"), on="x"]             # same
 DT[x=="a"]                     # same, single "==" internally optimised to use binary search (fast)
 DT[x!="b" | y!=3]              # not yet optimized, currently vector scan subset

man/last.Rd

@@ -18,6 +18,9 @@ of \code{xts::first} is deployed. }
 \item{\dots}{ Not applicable for \code{data.table} first/last. Any arguments here
 are passed through to \code{xts}'s first/last. }
 }
+\note{
+For zero-length vectors, \code{first(x)} and \code{last(x)} mimic \code{head(x, 1)} and \code{tail(x, 1)} by returning an empty vector instead of \code{NA}. However, unlike \code{head()}/\code{tail()} and base R subsetting (e.g., \code{x[1]}), they do not preserve attributes like names.
+}
 \value{
 If no other arguments are supplied it depends on the type of \code{x}. The first/last item
 of a vector or list. The first/last row of a \code{data.frame} or \code{data.table}.

man/setkey.Rd

@@ -74,6 +74,14 @@ The sort is \emph{stable}; i.e., the order of ties (if any) is preserved.
 For character vectors, \code{data.table} takes advantage of R's internal global string cache, also exported as \code{\link{chorder}}.
 }
 
+\section{Keys vs. Indices}{
+Setting a key (with \code{setkey}) and an index (with \code{setindex}) are similar, but have very important distinctions.
+
+Setting a key physically reorders the data in RAM.
+
+Setting an index computes the sort order, but instead of applying the reordering, simply \emph{stores} this computed ordering. That means that multiple indices can coexist, and that the original row order is preserved.
+}
+
 \section{Good practice}{
 In general, it's good practice to use column names rather than numbers. This is
 why \code{setkey} and \code{setkeyv} only accept column names.