Merge branch 'master' into frev

Author: ben-schwen

NEWS.md

@@ -22,6 +22,8 @@
 
 5. `transpose` gains `list.cols=` argument, [#5639](https://github.com/Rdatatable/data.table/issues/5639). Use this to return output with list columns and avoids type promotion (an exception is `factor` columns which are promoted to `character` for consistency between `list.cols=TRUE` and `list.cols=FALSE`). This is convenient for creating a row-major representation of a table. Thanks to @MLopez-Ibanez for the request, and Benjamin Schwendinger for the PR.
 
+4. Using `dt[, names(.SD) := lapply(.SD, fx)]` now works, [#795](https://github.com/Rdatatable/data.table/issues/795) -- one of our [most-requested issues (see #3189)](https://github.com/Rdatatable/data.table/issues/3189). Thanks to @brodieG for the report, 20 or so others for chiming in, and @ColeMiller1 for PR.
+
 ## BUG FIXES
 
 1. `unique()` returns a copy the case when `nrows(x) <= 1` instead of a mutable alias, [#5932](https://github.com/Rdatatable/data.table/pull/5932). This is consistent with existing `unique()` behavior when the input has no duplicates but more than one row. Thanks to @brookslogan for the report and @dshemetov for the fix.

R/data.table.R

@@ -1122,8 +1122,8 @@ replace_dot_alias = function(e) {
           if (is.name(lhs)) {
             lhs = as.character(lhs)
           } else {
-            # e.g. (MyVar):= or get("MyVar"):=
-            lhs = eval(lhs, parent.frame(), parent.frame())
+            # lhs is e.g. (MyVar) or get("MyVar") or names(.SD) || setdiff(names(.SD), cols)
+            lhs = eval(lhs, list(.SD = setNames(logical(length(sdvars)), sdvars)), parent.frame())
           }
         } else {
           # `:=`(c2=1L,c3=2L,...)

inst/tests/tests.Rraw

@@ -18352,53 +18352,92 @@ if (test_bit64) {
   test(2248, dcast(apple, id ~ time, value.var = "y"), data.table(id = c('a', 'b'), `1` = i64v[1:2], `2` = i64v[4:3], key='id'))
 }
 
+# Unit tests for DT[, .SD] retaining secondary indices, #1709
+DT = data.table(x=1:5, y=6:10)
+setindex(DT, x)
+test(2249.1, indices(DT), 'x')
+test(2249.2, indices(DT[, .SD]), 'x')
+setindex(DT, y)
+test(2249.3, indices(DT), c('x', 'y'))
+test(2249.4, indices(DT[, .SD]), c('x', 'y'))
+
+# make names(.SD) work - issue #795
+dt = data.table(a = 1:4, b = 5:8)
+test(2250.01, dt[, names(.SD) := lapply(.SD, '*', 2), .SDcols = 1L], data.table(a = 1:4 * 2, b = 5:8))
+test(2250.02, dt[, names(.SD) := lapply(.SD, '*', 2), .SDcols = 2L], data.table(a = 1:4 * 2, b = 5:8 * 2))
+test(2250.03, dt[, names(.SD) := lapply(.SD, as.integer)], data.table(a = as.integer(1:4 * 2), b = as.integer(5:8 * 2)))
+test(2250.04, dt[1L, names(.SD) := lapply(.SD, '+', 2L)], data.table(a = as.integer(c(4, 2:4 * 2)), b = as.integer(c(12, 6:8 * 2))))
+test(2250.05, dt[, setdiff(names(.SD), 'a') := NULL], data.table(a = as.integer(c(4, 2:4 * 2))))
+test(2250.06, dt[, c(names(.SD)) := NULL], null.data.table())
+
+dt = data.table(a = 1:4, b = 5:8, grp = c('a', 'a', 'b', 'c'))
+test(2250.07, dt[, names(.SD) := lapply(.SD, max), by = grp], data.table(a = c(2L, 2L, 3L, 4L), b = c(6L, 6L, 7L, 8L), grp = c('a', 'a', 'b', 'c')))
+
+dt = data.table(a = 1:4, b = 5:8, grp = c('a', 'a', 'b', 'c'))
+keep = c('a', 'b')
+test(2250.08, dt[, names(.SD) := NULL, .SDcols = !keep], data.table(a = 1:4, b = 5:8))
+
+dt = data.table(a = 1:4, b = 5:8, grp = c('a', 'a', 'b', 'c'))
+test(2250.09, dt[, paste(names(.SD), 'max', sep = '_') := lapply(.SD, max), by = grp] , data.table(a = 1:4, b = 5:8, grp = c('a', 'a', 'b', 'c'), a_max = c(2L, 2L, 3L, 4L), b_max = c(6L, 6L, 7L, 8L)))
+
+dt = data.table(a = 1:3, b = 5:7, grp = c('a', 'a', 'b'))
+test(2250.10, dt[1:2, paste(names(.SD), 'max', sep = '_') := lapply(.SD, max), by = grp], data.table(a = 1:3, b = 5:7, grp = c('a', 'a', 'b'), a_max = c(2L, 2L, NA_integer_), b_max = c(6L, 6L, NA_integer_)))
+test(2250.11, dt[, names(.SD(2)) := lapply(.SD, .I)], error = 'could not find function ".SD"')
+
+dt = data.table(a = 1:3, b = 5:7, grp = c('a', 'a', 'b'))
+test(2250.12, dt[, names(.SD) := lapply(.SD, \(x) x + b), .SDcols = "a"], data.table(a = 1:3 + 5:7, b = 5:7, grp = c('a', 'a', 'b')))
+
+
+dt = data.table(a = 1L, b = 2L, c = 3L, d = 4L, e = 5L, f = 6L)
+test(2250.13, dt[, names(.SD)[1:5] := sum(.SD)], data.table(a = 21L, b = 21L, c = 21L, d = 21L, e = 21L, f = 6L))
+
 # 5885 implement frev
 d = c(NA, NaN, Inf, -Inf)
-test(2249.00, frev(c(FALSE, NA)),     c(NA, FALSE))
-test(2249.01, frev(c(0L, NA)),        c(NA, 0L))
-test(2249.02, frev(d),                c(-Inf, Inf, NaN, NA))
-test(2249.03, frev(c(NA, 1, 0+2i)),   c(0+2i, 1, NA))
-test(2249.04, frev(as.raw(0:1)),      as.raw(1:0))
-test(2249.05, frev(NULL),             NULL)
-test(2249.06, frev(character(5)),     character(5))
-test(2249.07, frev(integer(0)),       integer(0))
-test(2249.08, frev(list(1, "a")),     list("a", 1))
-test(2249.09, setrev(c(0L, NA)),      c(NA, 0L))
-test(2249.10, setrev(d),              c(-Inf, Inf, NaN, NA))
-test(2249.11, setrev(c(NA, 1, 0+2i)), c(0+2i, 1, NA))
-test(2249.12, setrev(as.raw(0:1)),    as.raw(1:0))
-test(2249.13, setrev(NULL),           NULL)
-test(2249.14, setrev(character(5)),   character(5))
-test(2249.15, setrev(integer(0)),     integer(0))
-test(2249.16, setrev(list(1, "a")),   list("a", 1))
-test(2249.17, frev(1:1e2),            rev(1:1e2))
+test(2251.00, frev(c(FALSE, NA)),     c(NA, FALSE))
+test(2251.01, frev(c(0L, NA)),        c(NA, 0L))
+test(2251.02, frev(d),                c(-Inf, Inf, NaN, NA))
+test(2251.03, frev(c(NA, 1, 0+2i)),   c(0+2i, 1, NA))
+test(2251.04, frev(as.raw(0:1)),      as.raw(1:0))
+test(2251.05, frev(NULL),             NULL)
+test(2251.06, frev(character(5)),     character(5))
+test(2251.07, frev(integer(0)),       integer(0))
+test(2251.08, frev(list(1, "a")),     list("a", 1))
+test(2251.09, setrev(c(0L, NA)),      c(NA, 0L))
+test(2251.10, setrev(d),              c(-Inf, Inf, NaN, NA))
+test(2251.11, setrev(c(NA, 1, 0+2i)), c(0+2i, 1, NA))
+test(2251.12, setrev(as.raw(0:1)),    as.raw(1:0))
+test(2251.13, setrev(NULL),           NULL)
+test(2251.14, setrev(character(5)),   character(5))
+test(2251.15, setrev(integer(0)),     integer(0))
+test(2251.16, setrev(list(1, "a")),   list("a", 1))
+test(2251.17, frev(1:1e2),            rev(1:1e2))
 # copy arguments
 x = 1:3
-test(2249.21, {frev(x);  x}, 1:3)
-test(2249.22, {setrev(x); x}, 3:1)
-test(2249.23, address(x) == address(setrev(x)))
-test(2249.24, address(x) != address(frev(x)))
+test(2251.21, {frev(x);  x}, 1:3)
+test(2251.22, {setrev(x); x}, 3:1)
+test(2251.23, address(x) == address(setrev(x)))
+test(2251.24, address(x) != address(frev(x)))
 # do not alter on subsets
-test(2249.25, {setrev(x[1:2]);  x}, 1:3)
+test(2251.25, {setrev(x[1:2]);  x}, 1:3)
 # levels
 f = as.factor(letters)
-test(2249.31, frev(f), rev(f))
-test(2249.32, frev(as.IDate(1:10)), as.IDate(10:1))
-test(2249.33, frev(as.IDate(1:10)), as.IDate(10:1))
+test(2251.31, frev(f), rev(f))
+test(2251.32, frev(as.IDate(1:10)), as.IDate(10:1))
+test(2251.33, frev(as.IDate(1:10)), as.IDate(10:1))
 # names
 x = c(a=1L, b=2L, c=3L)
-test(2249.41, frev(x), rev(x))
-test(2249.42, setrev(x), x)
+test(2251.41, frev(x), rev(x))
+test(2251.42, setrev(x), x)
 # attributes
 x = structure(1:10, class = c("IDate", "Date"), att = 1L)
-test(2249.51, attr(frev(x), "att"), 1L)
-test(2249.52, attr(setrev(x), "att"), 1L)
+test(2251.51, attr(frev(x), "att"), 1L)
+test(2251.52, attr(setrev(x), "att"), 1L)
 # errors
-test(2249.61, frev(data.table()), error="should not be data.frame or data.table")
-test(2249.62, frev(expression(1)), error="is not supported by frev")
-test(2249.63, frev(matrix(1)), error="should not be matrix or array")
+test(2251.61, frev(data.table()), error="should not be data.frame or data.table")
+test(2251.62, frev(expression(1)), error="is not supported by frev")
+test(2251.63, frev(matrix(1)), error="should not be matrix or array")
 if (test_bit64) {
   x = as.integer64(c(1, NA, 3))
-  test(2249.71, frev(x), rev(x))
-  test(2249.72, setrev(x), x)
+  test(2251.71, frev(x), rev(x))
+  test(2251.72, setrev(x), x)
 }

man/assign.Rd

@@ -26,6 +26,9 @@
 #            LHS2 = RHS2,
 #            ...), by = ...]
 
+# 3. Multiple columns in place
+# DT[i, names(.SD) := lapply(.SD, fx), by = ..., .SDcols = ...]
+
 set(x, i = NULL, j, value)
 }
 \arguments{

vignettes/datatable-reference-semantics.Rmd

@@ -258,6 +258,23 @@ flights[, c("speed", "max_speed", "max_dep_delay", "max_arr_delay") := NULL]
 head(flights)
 ```
 
+#### -- How can we update multiple existing columns in place using `.SD`?
+
+```{r}
+flights[, names(.SD) := lapply(.SD, as.factor), .SDcols = is.character]
+```
+Let's clean up again and convert our newly-made factor columns back into character columns. This time we will make use of `.SDcols` accepting a function to decide which columns to include. In this case, `is.factor()` will return the columns which are factors. For more on the **S**ubset of the **D**ata, there is also an [SD Usage vignette](https://cran.r-project.org/web/packages/data.table/vignettes/datatable-sd-usage.html).
+
+Sometimes, it is also nice to keep track of columns that we transform. That way, even after we convert our columns we would be able to call the specific columns we were updating.
+```{r}
+factor_cols <- sapply(flights, is.factor)
+flights[, names(.SD) := lapply(.SD, as.character), .SDcols = factor_cols]
+str(flights[, ..factor_cols])
+```
+#### {.bs-callout .bs-callout-info}
+
+* We also could have used `(factor_cols)` on the `LHS` instead of `names(.SD)`.
+
 ## 3. `:=` and `copy()`
 
 `:=` modifies the input object by reference. Apart from the features we have discussed already, sometimes we might want to use the update by reference feature for its side effect. And at other times it may not be desirable to modify the original object, in which case we can use `copy()` function, as we will see in a moment.

vignettes/datatable-sd-usage.Rmd

@@ -77,7 +77,15 @@ The first way to impact what `.SD` is is to limit the _columns_ contained in `.S
 Pitching[ , .SD, .SDcols = c('W', 'L', 'G')]
 ```
 
-This is just for illustration and was pretty boring. But even this simply usage lends itself to a wide variety of highly beneficial / ubiquitous data manipulation operations:
+This is just for illustration and was pretty boring. In addition to accepting a character vector, `.SDcols` also accepts:
+
+1. any function such as `is.character` to filter _columns_
+2. the function^{*} `patterns()` to filter _column names_ by regular expression
+3. integer and logical vectors
+
+*see `?patterns` for more details
+
+This simple usage lends itself to a wide variety of highly beneficial / ubiquitous data manipulation operations:
 
 ## Column Type Conversion
 
@@ -91,52 +99,40 @@ We notice that the following columns are stored as `character` in the `Teams` da
 # teamIDretro: Team ID used by Retrosheet
 fkt = c('teamIDBR', 'teamIDlahman45', 'teamIDretro')
 # confirm that they're stored as `character`
-Teams[ , sapply(.SD, is.character), .SDcols = fkt]
+str(Teams[ , ..fkt])
 ```
 
-If you're confused by the use of `sapply` here, note that it's quite similar for base R `data.frames`:
-
-```{r identify_factors_as_df}
-setDF(Teams) # convert to data.frame for illustration
-sapply(Teams[ , fkt], is.character)
-setDT(Teams) # convert back to data.table
-```
-
-The key to understanding this syntax is to recall that a `data.table` (as well as a `data.frame`) can be considered as a `list` where each element is a column -- thus, `sapply`/`lapply` applies the `FUN` argument (in this case, `is.character`) to each _column_ and returns the result as `sapply`/`lapply` usually would.
-
-The syntax to now convert these columns to `factor` is very similar -- simply add the `:=` assignment operator:
+The syntax to now convert these columns to `factor` is simple:
 
 ```{r assign_factors}
-Teams[ , (fkt) := lapply(.SD, factor), .SDcols = fkt]
+Teams[ , names(.SD) := lapply(.SD, factor), .SDcols = patterns('teamID')]
 # print out the first column to demonstrate success
 head(unique(Teams[[fkt[1L]]]))
 ```
 
-Note that we must wrap `fkt` in parentheses `()` to force `data.table` to interpret this as column names, instead of trying to assign a column named `'fkt'`.
+Note: 
 
-Actually, the `.SDcols` argument is quite flexible; above, we supplied a `character` vector of column names. In other situations, it is more convenient to supply an `integer` vector of column _positions_ or a `logical` vector dictating include/exclude for each column. `.SDcols` even accepts regular expression-based pattern matching.
+1. The `:=` is an assignment operator to update the `data.table` in place without making a copy. See [reference semantics](https://cran.r-project.org/web/packages/data.table/vignettes/datatable-reference-semantics.html) for more. 
+2. The LHS, `names(.SD)`, indicates which columns we are updating - in this case we update the entire `.SD`.
+3. The RHS, `lapply()`, loops through each column of the `.SD` and converts the column to a factor.
+4. We use the `.SDcols` to only select columns that have pattern of `teamID`.
+
+Again, the `.SDcols` argument is quite flexible; above, we supplied `patterns` but we could have also supplied `fkt` or any `character` vector of column names. In other situations, it is more convenient to supply an `integer` vector of column _positions_ or a `logical` vector dictating include/exclude for each column. Finally, the use of a function to filter columns is very helpful.
 
 For example, we could do the following to convert all `factor` columns to `character`:
 
 ```{r sd_as_logical}
-# while .SDcols accepts a logical vector,
-#   := does not, so we need to convert to column
-#   positions with which()
-fkt_idx = which(sapply(Teams, is.factor))
-Teams[ , (fkt_idx) := lapply(.SD, as.character), .SDcols = fkt_idx]
-head(unique(Teams[[fkt_idx[1L]]]))
+fct_idx = Teams[, which(sapply(.SD, is.factor))] # column numbers to show the class changing
+str(Teams[[fct_idx[1L]]])
+Teams[ , names(.SD) := lapply(.SD, as.character), .SDcols = is.factor]
+str(Teams[[fct_idx[1L]]])
 ```
 
 Lastly, we can do pattern-based matching of columns in `.SDcols` to select all columns which contain `team` back to `factor`:
 
 ```{r sd_patterns}
 Teams[ , .SD, .SDcols = patterns('team')]
-
-# now convert these columns to factor;
-#   value = TRUE in grep() is for the LHS of := to
-#   get column names instead of positions
-team_idx = grep('team', names(Teams), value = TRUE)
-Teams[ , (team_idx) := lapply(.SD, factor), .SDcols = team_idx]
+Teams[ , names(.SD) := lapply(.SD, factor), .SDcols = patterns('team')]
 ```
 
 ** A proviso to the above: _explicitly_ using column numbers (like `DT[ , (1) := rnorm(.N)]`) is bad practice and can lead to silently corrupted code over time if column positions change. Even implicitly using numbers can be dangerous if we don't keep smart/strict control over the ordering of when we create the numbered index and when we use it.