Merge branch 'master' into issue6888

Author: venom1204

NEWS.md

@@ -6,6 +6,8 @@
 
 1. New `sort_by()` method for data.tables, [#6662](https://github.com/Rdatatable/data.table/issues/6662). It uses `forder()` to improve upon the data.frame method and also match `DT[order(...)]` behavior with respect to locale. Thanks @rikivillalba for the suggestion and PR.
 
+2. `melt()` now supports using `patterns()` with `id.vars`, [#6867](https://github.com/Rdatatable/data.table/issues/6867). Thanks to Toby Dylan Hocking for the suggestion and PR.
+
 ## BUG FIXES
 
 1. Custom binary operators from the `lubridate` package now work with objects of class `IDate` as with a `Date` subclass, [#6839](https://github.com/Rdatatable/data.table/issues/6839). Thanks @emallickhossain for the report and @aitap for the fix.
@@ -20,6 +22,9 @@
 
 6. By-reference sub-assignments to factor columns now match the levels in UTF-8, preventing their duplication when the same level exists in different encodings, [#6886](https://github.com/Rdatatable/data.table/issues/6886). Thanks @iagogv3 for the report and @aitap for the fix.
 
+7. `fwrite()` now avoids a crash when translating strings into a different encoding, [#6883](https://github.com/Rdatatable/data.table/issues/6883). Thanks @filipemsc for the report and @aitap for the fix.
+
+
 ## NOTES
 
 1. Continued work to remove non-API C functions, [#6180](https://github.com/Rdatatable/data.table/issues/6180). Thanks Ivan Krylov for the PRs and for writing a clear and concise guide about the R API: https://aitap.codeberg.page/R-api/.

R/fmelt.R

@@ -182,13 +182,17 @@ melt.data.table = function(data, id.vars, measure.vars, variable.name = "variabl
        value.name = "value", ..., na.rm = FALSE, variable.factor = TRUE, value.factor = FALSE,
        verbose = getOption("datatable.verbose")) {
   if (!is.data.table(data)) stopf("'data' must be a data.table")
-  if (missing(id.vars)) id.vars=NULL
-  if (missing(measure.vars)) measure.vars = NULL
-  measure.sub = substitute(measure.vars)
-  if (is.call(measure.sub)) {
-    eval.result = eval_with_cols(measure.sub, names(data))
-    if (!is.null(eval.result)) {
-      measure.vars = eval.result
+  for(type.vars in c("id.vars","measure.vars")){
+    sub.lang <- substitute({
+      if (missing(VAR)) VAR=NULL
+      substitute(VAR)
+    }, list(VAR=as.symbol(type.vars)))
+    sub.result = eval(sub.lang)
+    if (is.call(sub.result)) {
+      eval.result = eval_with_cols(sub.result, names(data))
+      if (!is.null(eval.result)) {
+        assign(type.vars, eval.result)
+      }
     }
   }
   if (is.list(measure.vars)) {

R/fwrite.R

@@ -111,6 +111,15 @@ fwrite = function(x, file="", append=FALSE, quote="auto",
   }
   # nocov end
   file = enc2native(file) # CfwriteR cannot handle UTF-8 if that is not the native encoding, see #3078.
+  # pre-encode any strings or factor levels to avoid translateChar trying to allocate from OpenMP threads
+  if (encoding %chin% c("UTF-8", "native")) {
+    enc = switch(encoding, "UTF-8" = enc2utf8, "native" = enc2native)
+    x = lapply(x, function(x) {
+      if (is.character(x)) x = enc(x)
+      if (is.factor(x)) levels(x) = enc(levels(x))
+      x
+    })
+  }
   .Call(CfwriteR, x, file, sep, sep2, eol, na, dec, quote, qmethod=="escape", append,
         row.names, col.names, logical01, scipen, dateTimeAs, buffMB, nThread,
         showProgress, is_gzip, compressLevel, bom, yaml, verbose, encoding)

inst/tests/tests.Rraw

@@ -3220,6 +3220,10 @@ test(1034, as.data.table(x<-as.character(sample(letters, 5))), data.table(V1=x))
   test(1035.12, attr(melt(DT, id.vars=1:2)$x, "foo"), "bla1")
   test(1035.13, attr(melt(DT, id.vars=1:2)$y, "bar"), 1:4)
 
+  # issue #6867 - id.vars=patterns().
+  DT=data.table(x_long=0, x_short=0, z=0, y1=1, y2=2)
+  test(1035.131, melt(DT, measure.vars=patterns("y"), id.vars=patterns("x")), data.table(x_long=0, x_short=0, variable=factor(c("y1","y2")), value=c(1,2)))
+
   # bug #699 - melt segfaults when vars are not in dt; was test 1316
   x = data.table(a=c(1,2),b=c(2,3),c=c(3,4))
   test(1035.14, melt(x, id.vars="d"), error="One or more values")
@@ -21121,3 +21125,7 @@ DT[1, V1 := samelevel]
 test(2311.1, nlevels(DT$V1), 1L) # used to be 2
 DT[1, V1 := factor("a", levels = c("a", samelevel))]
 test(2311.2, nlevels(DT$V1), 2L) # used to be 3
+
+# avoid translateChar*() in OpenMP threads, #6883
+DF = list(rep(iconv("\uf8", from = "UTF-8", to = "latin1"), 2e6))
+test(2312, fwrite(DF, nullfile(), encoding = "UTF-8", nThread = 2L), NULL)

man/melt.data.table.Rd

@@ -19,7 +19,7 @@ multiple columns simultaneously.
 \arguments{
 \item{data}{ A \code{data.table} object to melt.}
 \item{id.vars}{vector of id variables. Can be integer (corresponding id
-column numbers) or character (id column names) vector. If missing, all
+column numbers) or character (id column names) vector, perhaps created using \code{patterns()}. If missing, all
 non-measure columns will be assigned to it. If integer, must be positive; see Details. }
 \item{measure.vars}{Measure variables for \code{melt}ing. Can be missing, vector, list, or pattern-based.
 
@@ -131,6 +131,7 @@ melt(DT, id.vars=1, measure.vars=c("c_1", "c_2"), na.rm=TRUE) # remove NA
 # melt "f_1,f_2" and "d_1,d_2" simultaneously, retain 'factor' attribute
 # convenient way using internal function patterns()
 melt(DT, id.vars=1:2, measure.vars=patterns("^f_", "^d_"), value.factor=TRUE)
+melt(DT, id.vars=patterns("[in]"), measure.vars=patterns("^f_", "^d_"), value.factor=TRUE)
 # same as above, but provide list of columns directly by column names or indices
 melt(DT, id.vars=1:2, measure.vars=list(3:4, c("d_1", "d_2")), value.factor=TRUE)
 # same as above, but provide names directly: