Merge branch 'master' into todoRename

Author: badasahog

R/data.table.R

@@ -312,7 +312,7 @@ replace_dot_alias = function(e) {
         root = root_name(jsub)
       } else if (length(jsub) > 2L && jsub[[2L]] %iscall% ":=") {
         #2142 -- j can be {} and have length 1
-        stopf("You have wrapped := with {} which is ok but then := must be the only thing inside {}. You have something else inside {} as well. Consider placing the {} on the RHS of := instead; e.g. DT[,someCol:={tmpVar1<-...;tmpVar2<-...;tmpVar1*tmpVar2}]")
+        stopf("Invalid use of `:=` inside `{}`. `:=` must be the only expression inside `{}` when used in `j`. Instead of: DT[{tmp1 <- ...; tmp2 <- ...; someCol := tmp1 * tmp2}], Use: DT[, someCol := {tmp1 <- ...; tmp2 <- ...; tmp1 * tmp2}]")
       }
     }
     if (root=="eval" && !any(all.vars(jsub[[2L]]) %chin% names_x)) {
@@ -2884,7 +2884,7 @@ address = function(x) .Call(Caddress, eval(substitute(x), parent.frame()))
 
 ":=" = function(...) {
   # this error is detected when eval'ing isub and replaced with a more helpful one when using := in i due to forgetting a comma, #4227
-  stopf('Check that is.data.table(DT) == TRUE. Otherwise, :=, `:=`(...) and let(...) are defined for use in j, once only and in particular ways. Note that namespace-qualification like data.table::`:=`(...) is not supported. See help(":=").', class="dt_invalid_let_error")
+  stopf('Check that is.data.table(DT) == TRUE. Otherwise, `:=` is defined for use in j, once only and in particular ways. See help(":=", "data.table"). A common reason for this error is allocating a new column in `j` and using `<-` instead of `:=`; e.g., `DT[, new_col <- 1]` should be `DT[, new_col := 1]`. Another is using `:=` in a multi-statement `{...}` block; please use `:=` as the only statement in `j`.', class="dt_invalid_let_error")
 }
 
 # TODO(#6197): Export these.

R/fwrite.R

@@ -13,7 +13,8 @@ fwrite = function(x, file="", append=FALSE, quote="auto",
            yaml = FALSE,
            bom = FALSE,
            verbose=getOption("datatable.verbose", FALSE),
-           encoding = "") {
+           encoding = "",
+           forceDecimal = FALSE) {
   na = as.character(na[1L]) # fix for #1725
   if (length(encoding) != 1L || !encoding %chin% c("", "UTF-8", "native")) {
     stopf("Argument 'encoding' must be '', 'UTF-8' or 'native'.")
@@ -51,7 +52,7 @@ fwrite = function(x, file="", append=FALSE, quote="auto",
     length(compressLevel) == 1L && 0L <= compressLevel && compressLevel <= 9L,
     isTRUEorFALSE(col.names), isTRUEorFALSE(append), isTRUEorFALSE(row.names),
     isTRUEorFALSE(verbose), isTRUEorFALSE(showProgress), isTRUEorFALSE(logical01),
-    isTRUEorFALSE(bom),
+    isTRUEorFALSE(bom), isTRUEorFALSE(forceDecimal),
     length(na) == 1L, #1725, handles NULL or character(0) input
     is.character(file) && length(file)==1L && !is.na(file),
     length(buffMB)==1L && !is.na(buffMB) && 1L<=buffMB && buffMB<=1024L,
@@ -122,7 +123,7 @@ fwrite = function(x, file="", append=FALSE, quote="auto",
   }
   .Call(CfwriteR, x, file, sep, sep2, eol, na, dec, quote, qmethod=="escape", append,
         row.names, col.names, logical01, scipen, dateTimeAs, buffMB, nThread,
-        showProgress, is_gzip, compressLevel, bom, yaml, verbose, encoding)
+        showProgress, is_gzip, compressLevel, bom, yaml, verbose, encoding, forceDecimal)
   invisible()
 }
 

inst/tests/tests.Rraw

@@ -3997,9 +3997,9 @@ test(1133.8, DT[, new := if (.GRP==1L) 7L else 3.4, by=x], data.table(x=INT(1,1,
 DT <- data.table(x=c("A", "A", "B", "B"), val =1:4)
 DT2 <- copy(DT)[, a := 1L]
 test(1134.1, DT[, {a := 1L}], DT2)
-test(1134.2, DT[, {a := 1L; NULL}], error="You have wrapped.*which is ok.*Consider")
+test(1134.2, DT[, {a := 1L; NULL}], error="Invalid use of `:=` inside `{}`")
 test(1134.3, DT[, {b := 2L}, by=x], DT2[, b:=2L, by=x])
-test(1134.4, DT[, {b := 2L; sum(val)}, by=x], error="You have wrapped.*which is ok.*Consider")
+test(1134.4, DT[, {b := 2L; sum(val)}, by=x], error="Invalid use of `:=` inside `{}`")
 
 # FR #2693 and Gabor's suggestions on datatable-help "Problem with FAQ 2.8"
 d1 <- data.table(id1 = c(1L, 2L, 2L, 3L), val = 1:4, key = "id1")
@@ -21602,3 +21602,21 @@ if (getRversion() >= "4.0.0") { # rely on stopifnot(named = ...) for correct mes
 test(2337.3, is.null(fwrite(data.table(c(0.1, 0.2)), dec=",", sep="\t")))
 test(2337.4, is.null(fwrite(data.table(a=numeric(), b=numeric()), dec=",", sep=",")))
 test(2337.5, is.null(fwrite(data.table(a=numeric()), dec=",", sep=",")))
+
+# 2864 force decimal points for whole numbers in numeric columns
+dd = data.table(x=c(1, 2, 3))
+di = data.table(x=c(1L, 2L, 3L))
+test(2338.1, capture.output(fwrite(dd, forceDecimal=TRUE)), c("x", "1.", "2.", "3."))
+test(2338.2, capture.output(fwrite(dd, forceDecimal=TRUE, dec=",", sep="\t")), c("x", "1,", "2,", "3,"))
+test(2338.3, capture.output(fwrite(dd, forceDecimal=FALSE)), c("x", "1", "2", "3"))
+test(2338.4, capture.output(fwrite(di, forceDecimal=TRUE)), c("x", "1", "2", "3"))
+test(2338.5, capture.output(fwrite(data.table(x=c(0.)), forceDecimal=TRUE)), c("x", "0."))
+test(2338.6, capture.output(fwrite(data.table(x=c(-0.)), forceDecimal=TRUE)), c("x", "0."))
+test(2338.7, capture.output(fwrite(data.table(x=c(0.00)), forceDecimal=TRUE)), c("x", "0."))
+# round trip
+local({
+  f <- tempfile(); on.exit(unlink(f))
+  test(2338.8, {fwrite(dd, f, forceDecimal=TRUE); fread(f)}, dd)
+  test(2338.9, {fwrite(dd, f, forceDecimal=FALSE); fread(f)}, di)
+})
+

man/assign.Rd

@@ -6,6 +6,8 @@
 \description{
     Fast add, remove and update subsets of columns, by reference. \code{:=} operator can be used in two ways: \code{LHS := RHS} form, and \code{Functional form}. See \code{Usage}.
 
+    Note that when using \samp{:=} inside a \code{{...}} block in \code{j}, the \samp{:=} call must be the only statement. For assigning to multiple columns, use the functional form: \code{DT[, `:=`(col1=val1, col2=val2)]}. 
+
     \code{set} is a low-overhead loop-able version of \code{:=}. It is particularly useful for repetitively updating rows of certain columns by reference (using a for-loop). See \code{Examples}. It can not perform grouping operations.
 
     \code{let} is an alias for the functional form and behaves exactly like \code{`:=`}.
@@ -66,7 +68,8 @@ All of the following result in a friendly error (by design) :
     x := 1L
     DT[i, col] := val
     DT[i]$col := val
-    DT[, {col1 := 1L; col2 := 2L}]                # Use the functional form, `:=`(), instead (see above).
+    DT[, {col1 := 1L; col2 := 2L}]                # Using `{}` in `j` is reserved for single `:=` expressions.
+                                                  # For multiple updates, use the functional form `:=`() instead.
 }
 
 For additional resources, please read \href{../doc/datatable-faq.html}{\code{vignette("datatable-faq")}}. Also have a look at StackOverflow's \href{https://stackoverflow.com/questions/tagged/data.table/}{data.table tag}.

man/fwrite.Rd

@@ -21,7 +21,8 @@ fwrite(x, file = "", append = FALSE, quote = "auto",
   yaml = FALSE,
   bom = FALSE,
   verbose = getOption("datatable.verbose", FALSE),
-  encoding = "")
+  encoding = "",
+  forceDecimal = FALSE)
 }
 \arguments{
   \item{x}{Any \code{list} of same length vectors; e.g. \code{data.frame} and \code{data.table}. If \code{matrix}, it gets internally coerced to \code{data.table} preserving col names but not row names}
@@ -62,6 +63,7 @@ fwrite(x, file = "", append = FALSE, quote = "auto",
   \item{bom}{If \code{TRUE} a BOM (Byte Order Mark) sequence (EF BB BF) is added at the beginning of the file; format 'UTF-8 with BOM'.}
   \item{verbose}{Be chatty and report timings?}
   \item{encoding}{ The encoding of the strings written to the CSV file. Default is \code{""}, which means writing raw bytes without considering the encoding. Other possible options are \code{"UTF-8"} and \code{"native"}. }
+  \item{forceDecimal}{ Should decimal points be forced for whole numbers in numeric columns? When \code{FALSE}, the default, whole numbers like \code{c(1.0, 2.0, 3.0)} will be written as \samp{1, 2, 3} i.e., dropping \code{dec}. }
 }
 \details{
 \code{fwrite} began as a community contribution with \href{https://github.com/Rdatatable/data.table/pull/1613}{pull request #1613} by Otto Seiskari. This gave Matt Dowle the impetus to specialize the numeric formatting and to parallelize: \url{https://h2o.ai/blog/2016/fast-csv-writing-for-r/}. Final items were tracked in \href{https://github.com/Rdatatable/data.table/issues/1664}{issue #1664} such as automatic quoting, \code{bit64::integer64} support, decimal/scientific formatting exactly matching \code{write.csv} between 2.225074e-308 and 1.797693e+308 to 15 significant figures, \code{row.names}, dates (between 0000-03-01 and 9999-12-31), times and \code{sep2} for \code{list} columns where each cell can itself be a vector.

src/data.table.h

@@ -319,7 +319,7 @@ SEXP chmatch_R(SEXP, SEXP, SEXP);
 SEXP chmatchdup_R(SEXP, SEXP, SEXP);
 SEXP chin_R(SEXP, SEXP);
 SEXP freadR(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP);
-SEXP fwriteR(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP);
+SEXP fwriteR(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP);
 SEXP rbindlist(SEXP, SEXP, SEXP, SEXP, SEXP);
 SEXP setlistelt(SEXP, SEXP, SEXP);
 SEXP setS4elt(SEXP, SEXP, SEXP);

src/fwrite.c

@@ -42,6 +42,7 @@ static int scipen;
 static bool squashDateTime=false;      // 0=ISO(yyyy-mm-dd) 1=squash(yyyymmdd)
 static bool verbose=false;
 static int gzip_level;
+static bool forceDecimal=false;       // force writing decimal points for numeric columns
 
 extern const char *getString(const void *, int64_t);
 extern int getStringLen(const void *, int64_t);
@@ -198,6 +199,9 @@ void writeFloat64(const void *col, int64_t row, char **pch)
     }
   } else if (x == 0.0) {
     *ch++ = '0';   // and we're done.  so much easier rather than passing back special cases
+    if (forceDecimal) {
+      *ch++ = dec;
+    }
   } else {
     if (x < 0.0) { *ch++ = '-'; x = -x; }  // and we're done on sign, already written. no need to pass back sign
     union { double d; uint64_t l; } u;
@@ -237,8 +241,13 @@ void writeFloat64(const void *col, int64_t row, char **pch)
     if (l % 10 >= 5) l += 10; // use the last digit to round
     l /= 10;
     if (l == 0) {
+      // # nocov start. Very likely not needed as such numbers (e.g. 2^-1075) likely not representable in R.
       if (*(ch - 1) == '-') ch--;
       *ch++ = '0';
+      if (forceDecimal) {
+        *ch++ = dec;
+      }
+      // # nocov end
     } else {
       // Count trailing zeros and therefore s.f. present in l
       int trailZero = 0;
@@ -257,6 +266,10 @@ void writeFloat64(const void *col, int64_t row, char **pch)
         if (sf > dr) width = sf + 1;                     // 1.234 and 123.4
         else { dl0 = 1; width = dr + 1 + dl0; }            // 0.1234, 0.0001234
       }
+      const bool isWholeNumber = (dr == 0 && dl0 >= 0);
+      if (forceDecimal && isWholeNumber) { 
+        width += 1; 
+      }
       // So:  3.1416 => l=31416, sf=5, exp=0     dr=4; dl0=0; width=6
       //      30460  => l=3046, sf=4, exp=4      dr=0; dl0=1; width=5
       //      0.0072 => l=72, sf=2, exp=-3       dr=4; dl0=1; width=6
@@ -269,6 +282,8 @@ void writeFloat64(const void *col, int64_t row, char **pch)
           while (dr && sf) { *ch-- = '0' + l % 10; l /= 10; dr--; sf--; }
           while (dr) { *ch-- = '0'; dr--; }
           *ch-- = dec;
+        } else if (forceDecimal && isWholeNumber) {
+          *ch-- = dec;
         }
         while (dl0) { *ch-- = '0'; dl0--; }
         while (sf) { *ch-- = '0' + l % 10; l /= 10; sf--; }
@@ -615,6 +630,7 @@ void fwriteMain(fwriteMainArgs args)
   int8_t quoteHeaders = args.doQuote;
   verbose = args.verbose;
   gzip_level = args.gzip_level;
+  forceDecimal = args.forceDecimal;
 
   size_t len;
   unsigned int crc;

src/fwrite.h

@@ -117,6 +117,7 @@ typedef struct fwriteMainArgs
   bool bom;
   const char *yaml;
   bool verbose;
+  bool forceDecimal;      // force writing decimal points for numeric columns
 } fwriteMainArgs;
 
 void fwriteMain(fwriteMainArgs args);

src/fwriteR.c

@@ -172,7 +172,8 @@ SEXP fwriteR(
   SEXP bom_Arg,
   SEXP yaml_Arg,
   SEXP verbose_Arg,
-  SEXP encoding_Arg
+  SEXP encoding_Arg,
+  SEXP forceDecimal_Arg    // TRUE|FALSE
   )
 {
   if (!isNewList(DF)) error(_("fwrite must be passed an object of type list; e.g. data.frame, data.table"));
@@ -183,6 +184,7 @@ SEXP fwriteR(
   args.bom = LOGICAL(bom_Arg)[0];
   args.yaml = CHAR(STRING_ELT(yaml_Arg, 0));
   args.verbose = LOGICAL(verbose_Arg)[0];
+  args.forceDecimal = LOGICAL(forceDecimal_Arg)[0];
   args.filename = CHAR(STRING_ELT(filename_Arg, 0));
   args.ncol = length(DF);
   if (args.ncol == 0) {

src/idatetime.c

@@ -1,9 +1,9 @@
 #include "data.table.h"
 
-#define YEARS400 146097
-#define YEARS100 36524
-#define YEARS4 1461
-#define YEARS1 365
+static const int YEARS400 = 146097;
+static const int YEARS100 = 36524;
+static const int YEARS4 = 1461;
+static const int YEARS1 = 365;
 
 typedef enum { YDAY, WDAY, MDAY, WEEK, MONTH, QUARTER, YEAR, YEARMON, YEARQTR} datetype;
 
@@ -124,37 +124,40 @@ void convertSingleDate(int x, datetype type, void *out)
 SEXP convertDate(SEXP x, SEXP type)
 {
     if (!isInteger(x)) error(_("x must be an integer vector"));
-    const int *ix = INTEGER(x);
+    const int *ix = INTEGER_RO(x);
     const int n = length(x);
     if (!isString(type) || length(type) != 1)
         internal_error(__func__, "invalid type for, should have been caught before"); // # nocov
-    datetype ctype=0;
+    datetype ctype = 0;
     bool ansint = true;
-    if (!strcmp(CHAR(STRING_ELT(type, 0)), "yday")) ctype = YDAY;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "wday")) ctype = WDAY;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "mday")) ctype = MDAY;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "week")) ctype = WEEK;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "month")) ctype = MONTH;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "quarter")) ctype = QUARTER;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "year")) ctype = YEAR;
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "yearmon")) { ctype = YEARMON; ansint = false; }
-    else if (!strcmp(CHAR(STRING_ELT(type, 0)), "yearqtr")) { ctype = YEARQTR; ansint = false; }
-    else internal_error(__func__, "invalid type for, should have been caught before"); // # nocov
-
-    SEXP ans;
+
+    const char* ctype_str = CHAR(STRING_ELT(type, 0));
+    if (!strcmp(ctype_str, "yday")) ctype = YDAY;
+    else if (!strcmp(ctype_str, "wday")) ctype = WDAY;
+    else if (!strcmp(ctype_str, "mday")) ctype = MDAY;
+    else if (!strcmp(ctype_str, "week")) ctype = WEEK;
+    else if (!strcmp(ctype_str, "month")) ctype = MONTH;
+    else if (!strcmp(ctype_str, "quarter")) ctype = QUARTER;
+    else if (!strcmp(ctype_str, "year")) ctype = YEAR;
+    else if (!strcmp(ctype_str, "yearmon")) { ctype = YEARMON; ansint = false; }
+    else if (!strcmp(ctype_str, "yearqtr")) { ctype = YEARQTR; ansint = false; }
+    else internal_error(__func__, "invalid type, should have been caught before"); // # nocov
+    
     if (ansint) {
-        ans = PROTECT(allocVector(INTSXP, n));
+        SEXP ans = PROTECT(allocVector(INTSXP, n));
         int *ansp = INTEGER(ans);
-        for (int i=0; i < n; ++i) {
+        for (int i = 0; i < n; i++) {
             convertSingleDate(ix[i], ctype, &ansp[i]);
         }
+        UNPROTECT(1);
+        return ans;
     } else {
-        ans = PROTECT(allocVector(REALSXP, n));
+        SEXP ans = PROTECT(allocVector(REALSXP, n));
         double *ansp = REAL(ans);
-        for (int i=0; i < n; ++i) {
+        for (int i = 0; i < n; i++) {
             convertSingleDate(ix[i], ctype, &ansp[i]);
         }
+        UNPROTECT(1);
+        return ans;
     }
-    UNPROTECT(1);
-    return ans;
 }