Guard against infinite S3 dispatch loop (#6744)

* Guard against infinite S3 dispatch loop

* Test of simpler class reversal

* fix tests

* nocov for apparently spurious miss

Author: MichaelChirico

R/as.data.table.R

@@ -214,6 +214,7 @@ as.data.table.list = function(x,
 }
 
 as.data.table.data.frame = function(x, keep.rownames=FALSE, key=NULL, ...) {
+  if (is.data.table(x)) return(as.data.table.data.table(x)) # S3 is weird, #6739. Also # nocov; this is tested in 2302.{2,3}, not sure why it doesn't show up in coverage.
   if (!identical(class(x), "data.frame")) return(as.data.table(as.data.frame(x)))
   if (!isFALSE(keep.rownames)) {
     # can specify col name to keep.rownames, #575; if it's the same as key,

inst/tests/tests.Rraw

@@ -20748,15 +20748,19 @@ test(2301.3, DT[order(a, -b, decreasing=c(TRUE, TRUE))], error="Mixing '-' with
 test(2301.4, DT[order(a, b, decreasing=c(TRUE, TRUE, FALSE))], error="decreasing= has length 3")
 
 # as.data.table should remove extra attributes from extended data.frames #5699
-x = data.frame(a=c(1,5,3), b=c(2,4,6))
+x = data.frame(a=c(1, 5, 3), b=c(2, 4, 6))
 class(x) = c("tbl", "data.frame")
 attr(x, "t1") = "a"
 as.data.frame.tbl = function(x) {
   attr(x, "t1") = NULL
   class(x) = "data.frame"
   x
 }
-test(2302, attr(as.data.table(y), "t1"), attr(as.data.frame(y), "t1"))
+test(2302.1, attr(as.data.table(y), "t1"), attr(as.data.frame(y), "t1"))
+class(x) = c("data.frame", "data.table", "data.frame") # #6739
+test(2302.2, attr(as.data.table(y), "t1"), attr(as.data.frame(y), "t1"))
+class(x) = c("data.frame", "data.table") # duplicate class is overkill
+test(2302.3, attr(as.data.table(y), "t1"), attr(as.data.frame(y), "t1"))
 
 # by=foo(KEY) does not retain key (no way to guarantee monotonic transformation), #5583
 DT = data.table(a=1:2, key='a')