add test

Author: MichaelChirico

R/data.table.R

@@ -1526,6 +1526,7 @@ replace_dot_alias = function(e) {
     if (length(byval) && length(byval[[1L]])) {
       if (!bysameorder && isFALSE(byindex)) {
         if (verbose) {last.started.at=proc.time();catf("Finding groups using forderv ... ");flush.console()}
+        browser()
         o__ = forderv(byval, sort=keyby, retGrp=TRUE)
         # The sort= argument is called sortGroups at C level. It's primarily for saving the sort of unique strings at
         # C level for efficiency when by= not keyby=. Other types also retain appearance order, but at byte level to

inst/tests/tests.Rraw

@@ -20689,3 +20689,8 @@ if (test_bit64) {
   test(2300.3, DT1[DT2, on='id'], error="Incompatible join types")
   test(2300.4, DT2[DT1, on='id'], error="Incompatible join types")
 }
+
+# by=foo(KEY) does not retain key (no way to guarantee monotonic transformation), #5583
+DT = data.table(a=1:2, key='a')
+test(2301.1, DT[, .N, by=.(b=rev(a))], data.table(b=2:1, N=1L))
+test(2301.2, DT[, .(N=1L), by=.(b=rev(a))], data.table(b=2:1, N=1L)) # ensure no interaction with GForce