Allow double-integer64 joins when double is in (integer32 , integer64] range

Author: MichaelChirico

NEWS.md

@@ -107,6 +107,8 @@ rowwiseDT(
 
 11. `tables()` now returns the correct size for data.tables over 2GiB, [#6607](https://github.com/Rdatatable/data.table/issues/6607). Thanks to @vlulla for the report and the PR.
 
+12. Joins of `integer64` and `double` columns succeed when the `double` column has lossless `integer64` representation, [#6625](https://github.com/Rdatatable/data.table/issues/6625). Previously, this only worked when the double column had lossless _32-bit_ integer representation. Thanks @MichaelChirico for the report and fix.
+
 ## NOTES
 
 1. Tests run again when some Suggests packages are missing, [#6411](https://github.com/Rdatatable/data.table/issues/6411). Thanks @aadler for the note and @MichaelChirico for the fix.

R/bmerge.R

@@ -92,10 +92,10 @@ bmerge = function(i, x, icols, xcols, roll, rollends, nomatch, mult, ops, verbos
     if (xclass=="integer64" || iclass=="integer64") {
       nm = c(iname, xname)
       if (xclass=="integer64") { w=i; wc=ic; wclass=iclass; } else { w=x; wc=xc; wclass=xclass; nm=rev(nm) }  # w is which to coerce
-      if (wclass=="integer" || (wclass=="double" && !isReallyReal(w[[wc]]))) {
-        if (verbose) catf("Coercing %s column %s%s to type integer64 to match type of %s.\n", wclass, nm[1L], if (wclass=="double") " (which contains no fractions)" else "", nm[2L])
+      if (wclass=="integer" || (wclass=="double" && !isReallyReal(w[[wc]], i64=TRUE))) {
+        if (verbose) catf("Coercing %s column %s%s to type integer64 to match type of %s.\n", wclass, nm[1L], if (wclass=="double") " (which has integer64 representation, e.g. no fractions)" else "", nm[2L])
         set(w, j=wc, value=bit64::as.integer64(w[[wc]]))
-      } else stopf("Incompatible join types: %s is type integer64 but %s is type double and contains fractions", nm[2L], nm[1L])
+      } else stopf("Incompatible join types: %s is type integer64 but %s is type double and cannot be coerced to integer64 (e.g. has fractions)", nm[2L], nm[1L])
     } else {
       # just integer and double left
       if (iclass=="double") {

R/wrappers.R

@@ -18,6 +18,6 @@ colnamesInt = function(x, cols, check_dups=FALSE, skip_absent=FALSE) .Call(Ccoln
 testMsg = function(status=0L, nx=2L, nk=2L) .Call(CtestMsgR, as.integer(status)[1L], as.integer(nx)[1L], as.integer(nk)[1L])
 
 isRealReallyInt = function(x) .Call(CisRealReallyIntR, x)
-isReallyReal = function(x) .Call(CisReallyReal, x)
+isReallyReal = function(x, i64=FALSE) .Call(CisReallyReal, x, i64)
 
 coerceAs = function(x, as, copy=TRUE) .Call(CcoerceAs, x, as, copy)

inst/tests/tests.Rraw

@@ -20596,3 +20596,15 @@ test(2295.3, is.data.table(d2))
 
 # #6588: .checkTypos used to give arbitrary strings to stopf as the first argument
 test(2296, d2[x %no such operator% 1], error = '%no such operator%')
+
+if (test_bit64) {
+  # Join to integer64 doesn't require integer32 representation, just integer64, #6625
+  i64_val = .Machine$integer.max + 1
+  DT1 = data.table(id = as.integer64(i64_val))
+  DT2 = data.table(id = i64_val)
+  test(2297.1, DT1[DT2, on='id', verbose=TRUE], DT2, output="has integer64 representation")
+  test(2297.2, DT2[DT1, on='id', verbose=TRUE], DT1, output="has integer64 representation")
+  DT2[, id := id+.01]
+  test(2297.3, DT1[DT2, on='id'], error="Incompatible join types")
+  test(2297.4, DT2[DT1, on='id'], error="Incompatible join types")
+}

src/data.table.h

@@ -245,7 +245,7 @@ bool within_int32_repres(double x);
 bool within_int64_repres(double x);
 bool isRealReallyInt(SEXP x);
 SEXP isRealReallyIntR(SEXP x);
-SEXP isReallyReal(SEXP x);
+SEXP isReallyReal(SEXP x, SEXP i64);
 bool allNA(SEXP x, bool errorForBadType);
 SEXP colnamesInt(SEXP x, SEXP cols, SEXP check_dups, SEXP skip_absent);
 bool INHERITS(SEXP x, SEXP char_);

src/utils.c

@@ -22,6 +22,17 @@ static R_xlen_t firstNonInt(SEXP x) {
   return i==n ? 0 : i+1;
 }
 
+static R_xlen_t firstNonInt64(SEXP x) {
+  R_xlen_t n=xlength(x), i=0;
+  const double *dx = REAL(x);
+  while (i<n &&
+         ( ISNA(dx[i]) ||
+         (within_int64_repres(dx[i]) && dx[i]==(int64_t)(dx[i])))) {
+    i++;
+  }
+  return i==n ? 0 : i+1;
+}
+
 bool isRealReallyInt(SEXP x) {
   return isReal(x) ? firstNonInt(x)==0 : false;
   // used to error if not passed type double but this needed extra is.double() calls in calling R code
@@ -32,9 +43,13 @@ SEXP isRealReallyIntR(SEXP x) {
   return ScalarLogical(isRealReallyInt(x));
 }
 
-SEXP isReallyReal(SEXP x) {
-  return ScalarInteger(isReal(x) ? firstNonInt(x) : 0);
-  // return the 1-based location of first element which is really real (i.e. not an integer) otherwise 0 (false)
+// return the 1-based location of first element which is really real (i.e. not an integer) otherwise 0 (false)
+SEXP isReallyReal(SEXP x, SEXP i64) {
+  if (LOGICAL(i64)[0]) {
+    return ScalarInteger(isReal(x) ? firstNonInt64(x) : 0);
+  } else {
+    return ScalarInteger(isReal(x) ? firstNonInt(x) : 0);
+  }
 }
 
 bool allNA(SEXP x, bool errorForBadType) {