Merge branch 'master' into todoRename

Author: badasahog

.dev/CRAN_Release.cmd

@@ -584,26 +584,29 @@ bunzip2 inst/tests/*.Rraw.bz2  # decompress *.Rraw again so as not to commit com
 # Many thanks!
 # Best, Tyson
 # ------------------------------------------------------------
-# DO NOT commit or push to GitHub. Leave 4 files (.dev/CRAN_Release.cmd, DESCRIPTION, NEWS and init.c) edited and not committed. Include these in a single and final bump commit below.
-# DO NOT even use a PR. Because PRs build binaries and we don't want any binary versions of even release numbers available from anywhere other than CRAN.
+
 # Leave milestone open with a 'release checks' issue open. Keep updating status there.
 # ** If on EC2, shutdown instance. Otherwise get charged for potentially many days/weeks idle time with no alerts **
 # If it's evening, SLEEP.
 # It can take a few days for CRAN's checks to run. If any issues arise, backport locally. Resubmit the same even version to CRAN.
 # CRAN's first check is automatic and usually received within an hour. WAIT FOR THAT EMAIL.
 # When CRAN's email contains "Pretest results OK pending a manual inspection" (or similar), or if not and it is known why not and ok, then bump dev.
 
-###### Bump dev for NON-PATCH RELEASE
-# 0. Close milestone to prevent new issues being tagged with it. The final 'release checks' issue can be left open in a closed milestone.
-# 1. Check that 'git status' shows 4 files in modified and uncommitted state: DESCRIPTION, NEWS.md, init.c and this .dev/CRAN_Release.cmd
-# 2. Bump minor version in DESCRIPTION to next odd number. Note that DESCRIPTION was in edited and uncommitted state so even number never appears in git.
-# 3. Add new heading in NEWS for the next dev version. Add "(submitted to CRAN on <today>)" on the released heading.
-# 4. Bump minor version in dllVersion() in init.c
-# 5. Bump 3 minor version numbers in Makefile
-# 6. Search and replace this .dev/CRAN_Release.cmd to update 1.16.99 to 1.16.99 inc below, 1.16.0 to 1.17.0 above, 1.15.0 to 1.16.0 below
-# 7. Another final gd to view all diffs using meld. (I have `alias gd='git difftool &> /dev/null'` and difftool meld: http://meldmerge.org/)
-# 8. Push to master with this consistent commit message: "1.17.0 on CRAN. Bump to 1.17.99"
-# 9. Take sha from the previous step and run `git tag 1.17.0 96c..sha..d77` then `git push origin 1.16.0` (not `git push --tags` according to https://stackoverflow.com/a/5195913/403310)
+###### After submission for NON-PATCH RELEASE
+# 0. Start a new branch `cran-x.y.0` with the code as submitted to CRAN
+#    - Check that 'git status' shows 4 files in modified and uncommitted state: DESCRIPTION, NEWS.md, init.c and this .dev/CRAN_Release.cmd
+#    - The branch should have one commit with precisely these 4 files being edited
+# 1. Follow up with a commit with this consistent commit message like: "1.17.0 on CRAN. Bump to 1.17.99" to this branch bumping to the next dev version
+#    - Bump minor version in DESCRIPTION to next odd number. Note that DESCRIPTION was in edited and uncommitted state so even number never appears in git.
+#    - Add new heading in NEWS for the next dev version. Add "(submitted to CRAN on <today>)" on the released heading.
+#    - Bump minor version in dllVersion() in init.c
+#    - Bump 3 minor version numbers in Makefile
+#    - Search and replace this .dev/CRAN_Release.cmd to update 1.16.99 to 1.16.99 inc below, 1.16.0 to 1.17.0 above, 1.15.0 to 1.16.0 below
+#    - Another final gd to view all diffs using meld. (I have `alias gd='git difftool &> /dev/null'` and difftool meld: http://meldmerge.org/)
+# 2. Ideally, no PRs are reviewed while a CRAN submission is pending. Any reviews that do happen MUST target this branch, NOT master!
+# 3. Once the submission lands on CRAN, merge this branch WITHOUT SQUASHING!
+# 4. Close milestone to prevent new issues being tagged with it. The final 'release checks' issue can be left open in a closed milestone.
+# 5. Take SHA from the "...on CRAN. Bump to ..." commit and run `git tag 1.17.0 96c..sha..d77` then `git push origin 1.17.0` (not `git push --tags` according to https://stackoverflow.com/a/5195913/403310)
 ######
 
 ###### Branching policy for PATCH RELEASE

.github/workflows/R-CMD-check.yaml

@@ -36,6 +36,7 @@ jobs:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       RSPM: ${{ matrix.config.rspm }}
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      _R_CHECK_RD_CHECKRD_MINLEVEL_: -Inf
 
     steps:
       - uses: actions/checkout@v4

.gitlab-ci.yml

@@ -114,6 +114,7 @@ test-lin-rel:
   variables:
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
     OPENBLAS_MAIN_FREE: "1"
+    _R_CHECK_RD_CHECKRD_MINLEVEL_: "-Inf"
   script:
     - *install-deps
     - echo 'CFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars

GOVERNANCE.md

@@ -108,7 +108,7 @@ Please also make a note in the change log under [`# Governance history`](#govern
 
 # Finances and Funding
 
-data.table is a [NumFOCUS](https://numfocus.org/) project.  Donations to the data.table can be made at [https://numfocus.org/project/data-table]([https://numfocus.org/donate-to-data-table](https://app.hubspot.com/payments/FFWKWTTvKFdzqH?referrer=PAYMENT_LINK))
+data.table is a [NumFOCUS](https://numfocus.org/) project. Donations to data.table can be made at [https://numfocus.org/project/data-table](https://app.hubspot.com/payments/FFWKWTTvKFdzqH?referrer=PAYMENT_LINK).
 
 *NumFOCUS is a 501(c)(3) non-profit charity in the United States; as such, donations to NumFOCUS are tax-deductible as allowed by law. As with any donation, you should consult with your personal tax adviser or the IRS about your particular tax situation.*
 

NAMESPACE

@@ -155,6 +155,7 @@ if (getRversion() >= "3.6.0") {
 export(as.IDate,as.ITime,IDateTime)
 export(second,minute,hour,yday,wday,mday,week,isoweek,isoyear,month,quarter,year,yearmon,yearqtr)
 
+if (getRversion() >= "4.3.0") S3method(chooseOpsMethod, IDate)
 S3method("[", ITime)
 S3method("+", IDate)
 S3method("-", IDate)

NEWS.md

@@ -101,6 +101,10 @@
 
 15. `split.data.table()` output list elements retain the S3 class of the generating data.table, e.g. in `l=split(x, ...)` if `x` has class `my_class`, so will `l[[1]]` and so on, [#7105](https://github.com/Rdatatable/data.table/issues/7105). Thanks @m-muecke for the bug report and @MichaelChirico for the fix.
 
+16. `between()` is now more robust with `integer64` arguments. Combining small integer `x` with certain large `integer64` bounds no longer misinterprets the bounds as `double`; if a `double` bound cannot be losslessly converted into `integer64` for comparison with `integer64` `x`, an error is signalled instead of returning a wrong answer with a warning; [#7164](https://github.com/Rdatatable/data.table/issues/7164). Thanks @aitap for the bug report and the fix.
+
+17. `t1 - t2`, where one is an `IDate` and the other is a `Date`, are now consistent with the case where both are `IDate` or both are `Date`, [#4749](https://github.com/Rdatatable/data.table/issues/4749). Thanks @George9000 for the report and @MichaelChirico for the fix.
+
 ### NOTES
 
 1. The following in-progress deprecations have proceeded:

R/IDateTime.R

@@ -91,6 +91,8 @@ round.IDate = function(x, digits=c("weeks", "months", "quarters", "years"), ...)
           years = ISOdate(year(x), 1L, 1L)))
 }
 
+chooseOpsMethod.IDate = function(x, y, mx, my, cl, reverse) inherits(y, "Date")
+
 #Adapted from `+.Date`
 `+.IDate` = function(e1, e2) {
   if (nargs() == 1L)
@@ -115,7 +117,7 @@ round.IDate = function(x, digits=c("weeks", "months", "quarters", "years"), ...)
   if (storage.mode(e1) != "integer")
     internal_error("storage mode of IDate is somehow no longer integer") # nocov
   if (nargs() == 1L)
-    stopf("unary - is not defined for \"IDate\" objects")
+    stopf('unary - is not defined for "IDate" objects')
   if (inherits(e2, "difftime"))
     internal_error("difftime objects may not be subtracted from IDate, but Ops dispatch should have intervened to prevent this") # nocov
 
@@ -127,7 +129,12 @@ round.IDate = function(x, digits=c("weeks", "months", "quarters", "years"), ...)
     # ii) .Date was newly exposed in R some time after 3.4.4
   }
   ans = as.integer(unclass(e1) - unclass(e2))
-  if (!inherits(e2, "Date")) setattr(ans, "class", c("IDate", "Date"))
+  if (inherits(e2, "Date")) {
+    setattr(ans, "class", "difftime")
+    setattr(ans, "units", "days")
+  } else {
+    setattr(ans, "class", c("IDate", "Date"))
+  }
   ans
 }
 

R/setops.R

@@ -160,8 +160,8 @@ all.equal.data.table = function(target, current, trim.levels=TRUE, check.attribu
       return(sprintf(
         "%s. 'target': %s. 'current': %s.",
         gettext("Datasets have different keys"),
-        if(length(k1)) brackify(k1) else gettextf("has no key"),
-        if(length(k2)) brackify(k2) else gettextf("has no key")
+        if(length(k1)) brackify(k1) else gettext("has no key"),
+        if(length(k2)) brackify(k2) else gettext("has no key")
       ))
     }
     # check index
@@ -171,8 +171,8 @@ all.equal.data.table = function(target, current, trim.levels=TRUE, check.attribu
       return(sprintf(
         "%s. 'target': %s. 'current': %s.",
         gettext("Datasets have different indices"),
-        if(length(i1)) brackify(i1) else gettextf("has no index"),
-        if(length(i2)) brackify(i2) else gettextf("has no index")
+        if(length(i1)) brackify(i1) else gettext("has no index"),
+        if(length(i2)) brackify(i2) else gettext("has no index")
       ))
     }
 

R/test.data.table.R

@@ -205,7 +205,30 @@ test.data.table = function(script="tests.Rraw", verbose=FALSE, pkg=".", silent=F
   }
   # nocov end
 
-  err = try(sys.source(fn, envir=env), silent=silent)
+  env$warnings = list()
+  err = try(
+    withCallingHandlers(
+      sys.source(fn, envir=env),
+      warning=function(w) {
+        # nocov start
+        if (!silent && showProgress) print(w)
+        env$warnings = c(env$warnings, list(list(
+          "after test"=env$prevtest, warning=conditionMessage(w),
+          calls=paste(
+            vapply_1c(sys.calls(), function(call) {
+              if (is.name(call[[1]])) {
+                as.character(call[[1]])
+              } else "..."
+            }),
+            collapse=" -> "
+          )
+        )))
+        invokeRestart("muffleWarning")
+        # nocov end
+      }
+    ),
+    silent=silent
+  )
 
   options(oldOptions)
   for (i in oldEnv) {
@@ -262,6 +285,21 @@ test.data.table = function(script="tests.Rraw", verbose=FALSE, pkg=".", silent=F
     # important to stopf() here, so that 'R CMD check' fails
     # nocov end
   }
+  if (length(env$warnings)) {
+    # nocov start
+    warnings = rbindlist(env$warnings)
+    catf(
+      ngettext(nrow(warnings),
+        "Caught %d warning outside the test() calls:\n",
+        "Caught %d warnings outside the test() calls:\n"
+      ),
+      nrow(warnings),
+      domain=NA
+    )
+    print(warnings, nrows = nrow(warnings))
+    stopf("Tests succeeded, but non-test code caused warnings. Search %s for tests shown above.", names(fn))
+    # nocov end
+  }
 
   # There aren't any errors, so we can use up 11 lines for the timings table
   time = nTest = RSS = NULL  # to avoid 'no visible binding' note

inst/tests/tests.Rraw

@@ -10342,7 +10342,7 @@ test(1673.1, TT + 4L, as.IDate("2016-04-29"))
 test(1673.2, TT + 4, as.IDate("2016-04-29"))
 test(1673.3, TT - 3, as.IDate("2016-04-22"))
 test(1673.4, TT - 3L, as.IDate("2016-04-22"))
-test(1673.5, as.IDate("2016-04-28") - as.IDate("2016-04-20"), 8L)
+test(1673.5, all.equal(as.IDate("2016-04-28") - as.IDate("2016-04-20"), as.difftime(8, units='days')))
 
 
 # test for radix integer order when MAXINT is present AND decreasing=TRUE AND na.last=FALSE
@@ -16996,7 +16996,8 @@ if (.Platform$OS.type=="windows") local({
     LC_TIME = "Chinese (Simplified)_China.936"
   )
   x_old = Map(Sys.getlocale, names(x))
-  invisible(Map(Sys.setlocale, names(x), x))
+  # setlocale() warns for non-UTF-8 locales in modern versions of R, #7210
+  suppressWarnings(invisible(Map(Sys.setlocale, names(x), x)))
   on.exit(Map(Sys.setlocale, names(x_old), x_old))
   # triggered segfault here in #4402, Windows-only under translation.
   # test that the argument order changes correctly (the 'item 2' moves to the beginning of the message)
@@ -18615,12 +18616,18 @@ test(2252.2, dt[, let(b=2L)], error = "\\[ was called on a data.table.*not data.
 rm(.datatable.aware)
 
 # tests for trunc.char handling wide characters # 5096
-local({
+local(if (l10n_info()$`UTF-8` || {
   lc_ctype = Sys.getlocale('LC_CTYPE')
+  lc_wantctype = 'en_US.UTF-8'
   # Japanese multibyte characters require utf8. As of 2025, we're likely to be already running in a UTF-8 locale, but if not, try this setlocale() call as a last chance.
   # Unfortunately, there is no guaranteed, portable way of switching to UTF-8 US English.
-  if (!l10n_info()$`UTF-8`) Sys.setlocale('LC_CTYPE', "en_US.UTF-8")
-  on.exit(Sys.setlocale('LC_CTYPE', lc_ctype))
+  # Avoid the warning upon possible failure, #7210.
+  lc_newctype = suppressWarnings(Sys.setlocale('LC_CTYPE', lc_wantctype))
+  if (identical(lc_newctype, lc_wantctype)) {
+    on.exit(Sys.setlocale('LC_CTYPE', lc_ctype))
+    TRUE
+  } else FALSE
+}) {
   accented_a = "\u0061\u0301"
   ja_ichi = "\u4E00"
   ja_ni = "\u4E8C"
@@ -18670,6 +18677,8 @@ local({
     paste0(c(ja_ko, ja_n, accented_a), dots, collapse=" ")))
   # test for data.table with NA, #6441
   test(2253.20, options=list(datatable.prettyprint.char = 1L), data.table(a = c("abc", NA)), output="      a\n1: a...\n2: <NA>")
+} else {
+  cat("Tests 2253* skipped because they need a UTF-8 locale.\n")
 })
 
 # allow 1-D matrix in j for consistency, #783
@@ -21554,10 +21563,18 @@ test(2332.81, {M1[]=frev(M1); M1}, {M2[]=rev(M2); M2})
 test(2333, as.expression(data.table(a = 1))[["a"]], 1)
 
 # regression test for hexdigits subscript overrun (uint8_t wraps over 255, unsigned overflow is well defined in c)
-f = tempfile()
-writeLines(c('a', rep('0x1.ffffp0', 10000L), '0x1.ff\x9fp0', rep('0x1.ffffp0', 20000L)), f)
-test(2334, names(fread(f)), "a")
-unlink(f)
+local({
+	f = tempfile()
+	on.exit(unlink(f))
+	# the line is likely invalid in current encoding, so disable any translation, #7209
+	# test.data.table() sets options(encoding="UTF-8"), so go the long way around.
+	ff = file(f, encoding = "")
+	tryCatch(
+		writeLines(c('a', rep('0x1.ffffp0', 10000L), `Encoding<-`('0x1.ff\x9fp0', 'bytes'), rep('0x1.ffffp0', 20000L)), ff),
+		finally = close(ff)
+	)
+	test(2334, names(fread(f)), "a")
+})
 
 # Tests for new isoyear() helper (complement to isoweek) #7154
 test(2335.1, isoyear(as.IDate("2019-12-30")), 2020L)  # End of year edge case
@@ -21566,3 +21583,13 @@ test(2335.3, isoyear(as.IDate("2023-08-15")), 2023L)  # Normal mid-year case
 test(2335.4, isoyear(as.IDate(c("2019-12-30", "2016-01-01", "2023-08-15"))),c(2020L, 2015L, 2023L))
 test(2335.5, isoyear("2019-12-30"), 2020L)
 test(2335.6, isoyear(as.Date("2019-12-30")), 2020L)
+
+# t1-t2 for Date/IDate should be consistent, modulo storage mode #4979
+t1 = as.IDate("2025-07-01")
+t2 = as.IDate("2025-06-01")
+test(2336.1, all.equal(as.Date(t1) - as.Date(t2), t1 - t2))
+test(2336.2, all.equal(as.Date(t2) - as.Date(t1), t2 - t1))
+test(2336.3, all.equal(as.Date(t1) - t2, t1 - t2))
+test(2336.4, all.equal(as.Date(t2) - t1, t2 - t1))
+test(2336.5, all.equal(t1 - as.Date(t2), t1 - t2))
+test(2336.6, all.equal(t2 - as.Date(t1), t2 - t1))