Merge branch 'master' into issue____2611

Author: venom1204

.github/workflows/pkgup.yaml

@@ -39,7 +39,6 @@ jobs:
           Rscript -e 'stopifnot(file.copy("DESCRIPTION", file.path(tdir<-tempdir(), "PACKAGES"))); db<-available.packages(paste0("file://", tdir)); deps<-setdiff(tools::package_dependencies(read.dcf("DESCRIPTION", fields="Package")[[1L]], db, which="most")[[1L]], installed.packages(priority="high")[,"Package"]); if (length(deps)) { ap<-available.packages()[,"Version"]; ap<-ap[names(ap) %in% deps]; if (!all(deps%in%names(ap))) stop("dependencies are not avaiable in repository: ",paste(setdiff(deps, names(ap)), collapse=", ")); ip<-installed.packages()[,"Version"]; ip<-ip[names(ip) %in% deps]; pkgs<-ap[deps]>ip[deps]; install.packages(names(pkgs[pkgs|is.na(pkgs)]), INSTALL_opts="--html") }'
       - name: build
         run: |
-          sed -i "0,/^Version: [0-9.]\+$/s//&-$(TZ=UTC git log -1 --format=%ct)/" ./DESCRIPTION
           echo "Revision:" $GITHUB_SHA >> ./DESCRIPTION
           R CMD build .
       - name: check

.gitlab-ci.yml

@@ -77,8 +77,6 @@ build:
     - rm -r bus
   script:
     - sed -i '/^[[:space:]]*$/d' ./DESCRIPTION ## make last line end abruptly; i.e. without a final \n
-    - |
-      sed -i "0,/^Version: [0-9.]\+$/s//&-$(TZ=UTC git log -1 --format=%ct)/" ./DESCRIPTION
     - echo "Revision:" $CI_COMMIT_SHA >> ./DESCRIPTION
     - R CMD build .
     - mkdir -p bus/$CI_JOB_NAME/
@@ -184,7 +182,7 @@ test-lin-dev-gcc-strict-cran:
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - (! grep "warning:" data.table.Rcheck/00install.out)
     - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls, V8 package) but ", shQuote(l)) else q("no")'
+        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 2 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (non-API calls, V8 package) but ", shQuote(l)) else q("no")'
 
 ## R-devel on Linux clang
 # R compiled with clang, flags removed: -flto=auto -fopenmp
@@ -207,7 +205,7 @@ test-lin-dev-clang-cran:
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - (! grep "warning:" data.table.Rcheck/00install.out)
     - >-
-        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls, V8 package) but ", shQuote(l)) else q("no")'
+        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 2 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (non-API calls, V8 package) but ", shQuote(l)) else q("no")'
 
 # stated dependency on R
 test-lin-ancient-cran:

R/onLoad.R

@@ -14,10 +14,6 @@
     # check when installed package is loaded but skip when developing the package with cc()
     dllV = if (is.loaded("CdllVersion",PACKAGE="data_table")) .Call(CdllVersion) else "before 1.12.0"
     RV = as.character(packageVersion("data.table"))
-    if (sum(.dots <- charToRaw(RV) == charToRaw(".")) > 2L) {
-      ## trim dev version suffix 1.17.99-1234567890, note that base:::package_version turns `-` into `.` #7339
-      RV = substring(RV, 1L, which(.dots)[3L]-1L)
-    }
     if (dllV != RV) {
       dll = if (.Platform$OS.type=="windows") "dll" else "so"
       # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17478

README.md

@@ -3,7 +3,7 @@
 
 <!-- badges: start -->
 [![CRAN status](https://badges.cranchecks.info/flavor/release/data.table.svg)](https://cran.r-project.org/web/checks/check_results_data.table.html)
-[![R-CMD-check](https://github.com/Rdatatable/data.table/workflows/R-CMD-check/badge.svg)](https://github.com/Rdatatable/data.table/actions)
+[![R-CMD-check](https://github.com/Rdatatable/data.table/actions/workflows/R-CMD-check.yaml/badge.svg?branch=master)](https://github.com/Rdatatable/data.table/actions)
 [![Codecov test coverage](https://codecov.io/github/Rdatatable/data.table/coverage.svg?branch=master)](https://app.codecov.io/github/Rdatatable/data.table?branch=master)
 [![GitLab CI build status](https://gitlab.com/Rdatatable/data.table/badges/master/pipeline.svg)](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html)
 [![downloads](https://cranlogs.r-pkg.org/badges/data.table)](https://www.rdocumentation.org/trends)

man/data.table.Rd

@@ -422,6 +422,10 @@ DT[, list(MySum=sum(v),
           MyMax=max(v)),
     by=.(x, y\%\%2)]                    # by 2 expressions
 
+# programmatic query with env=
+DT[, .(funvar = fun(var)), by=grp_var,
+   env = list(fun="sum", var="a", funvar="sum_a_by_x", grp_var="x")]
+
 DT[, .(a = .(a), b = .(b)), by=x]     # list columns
 DT[, .(seq = min(a):max(b)), by=x]    # j is not limited to just aggregations
 DT[, sum(v), by=x][V1<20]             # compound query

man/mergelist.Rd

@@ -113,6 +113,7 @@ l = list(
 lapply(l[-1L], `[`, j = if (.N>1L) .SD, by = "id1") ## duplicated rows
 try(mergelist(l, on="id1"))
 
+\donttest{
 ## 'star schema' and 'snowflake schema' examples (realistic data sizes)
 
 ### populate fact: US population by state and date
@@ -199,4 +200,5 @@ ans = mergelist(list(
   setmergelist(rev(geog), how="right")
 ))
 }
+}
 \keyword{ data }

src/fifelse.c

@@ -217,14 +217,14 @@ SEXP fcaseR(SEXP rho, SEXP args) {
   int nprotect=0, l;
   int64_t n_ans=0, n_this_arg=0, n_undecided=0;
   SEXP ans=R_NilValue, tracker=R_NilValue, whens=R_NilValue, thens=R_NilValue;
-  SEXP ans_class, ans_levels;
+  SEXP ans_class, ans_levels = R_NilValue;
   PROTECT_INDEX Iwhens, Ithens;
   PROTECT_WITH_INDEX(whens, &Iwhens); nprotect++;
   PROTECT_WITH_INDEX(thens, &Ithens); nprotect++;
   SEXPTYPE ans_type=NILSXP;
   // naout means if the output is scalar logic na
   bool imask = true, naout = false, idefault = false;
-  bool ans_is_factor;
+  bool ans_is_factor = false;
   int *restrict p = NULL;
   const int n = narg/2;
   for (int i=0; i<n; ++i) {

src/fread.c

@@ -1689,7 +1689,7 @@ int freadMain(freadMainArgs _args)
   if (ch >= eof) STOP(_("Input is either empty, fully whitespace, or skip has been set after the last non-whitespace."));
   if (verbose) {
     if (lineStart > ch) DTPRINT(_("  Moved forward to first non-blank line (%d)\n"), row1line);
-    DTPRINT(_("  Positioned on line %d starting: <<%s>>\n"), row1line, strlim(lineStart, (char[500]) {}, 30));
+    DTPRINT(_("  Positioned on line %d starting: <<%s>>\n"), row1line, strlim(lineStart, (char[500]) {0}, 30));
   }
   ch = pos = lineStart;
   }
@@ -1880,7 +1880,7 @@ int freadMain(freadMainArgs _args)
     if (!fill && tt != ncol) INTERNAL_STOP("first line has field count %d but expecting %d", tt, ncol); // # nocov
     if (verbose) {
       DTPRINT(_("  Detected %d columns on line %d. This line is either column names or first data row. Line starts as: <<%s>>\n"),
-              tt, row1line, strlim(pos, (char[500]) {}, 30));
+              tt, row1line, strlim(pos, (char[500]) {0}, 30));
       DTPRINT(_("  Quote rule picked = %d\n"), quoteRule);
       DTPRINT(_("  fill=%s and the most number of columns found is %d\n"), fill ? "true" : "false", ncol);
     }
@@ -2809,23 +2809,23 @@ int freadMain(freadMainArgs _args)
       while (ch < eof && *ch != '\n' && *ch != '\r') ch++;
       while (ch < eof && isspace(*ch)) ch++;
       if (ch == eof) {
-        DTWARN(_("Discarded single-line footer: <<%s>>"), strlim(skippedFooter, (char[500]) {}, 500));
+        DTWARN(_("Discarded single-line footer: <<%s>>"), strlim(skippedFooter, (char[500]) {0}, 500));
       }
       else {
         ch = headPos;
         int tt = countfields(&ch);
         if (fill > 0) {
           DTWARN(_("Stopped early on line %"PRId64". Expected %d fields but found %d. Consider fill=%d or even more based on your knowledge of the input file. Use fill=Inf for reading the whole file for detecting the number of fields. First discarded non-empty line: <<%s>>"),
-          DTi + row1line, ncol, tt, tt, strlim(skippedFooter, (char[500]) {}, 500));
+          DTi + row1line, ncol, tt, tt, strlim(skippedFooter, (char[500]) {0}, 500));
         } else {
           DTWARN(_("Stopped early on line %"PRId64". Expected %d fields but found %d. Consider fill=TRUE. First discarded non-empty line: <<%s>>"),
-          DTi + row1line, ncol, tt, strlim(skippedFooter, (char[500]) {}, 500));
+          DTi + row1line, ncol, tt, strlim(skippedFooter, (char[500]) {0}, 500));
         }
       }
     }
   }
   if (quoteRuleBumpedCh != NULL && quoteRuleBumpedCh < headPos) {
-    DTWARN(_("Found and resolved improper quoting out-of-sample. First healed line %"PRId64": <<%s>>. If the fields are not quoted (e.g. field separator does not appear within any field), try quote=\"\" to avoid this warning."), quoteRuleBumpedLine, strlim(quoteRuleBumpedCh, (char[500]) {}, 500));
+    DTWARN(_("Found and resolved improper quoting out-of-sample. First healed line %"PRId64": <<%s>>. If the fields are not quoted (e.g. field separator does not appear within any field), try quote=\"\" to avoid this warning."), quoteRuleBumpedLine, strlim(quoteRuleBumpedCh, (char[500]) {0}, 500));
   }
 
   if (verbose) {

vignettes/datatable-programming.Rmd

@@ -204,6 +204,33 @@ DT[filter_col %in% filter_val,
   )]
 ```
 
+### Substitute functions
+
+A tiny clarification may be useful on how we can substitute a function name in an expression.
+Note that providing `outer="sqrt"` (string) vs. `outer=sqrt` (symbol) is very different:
+```{r substitute_fun1, result='hide'}
+DT[, outer(Sepal.Length), env = list(outer="sqrt"), verbose=TRUE]
+#Argument 'j' after substitute: sqrt(Sepal.Length)
+## DT[, sqrt(Sepal.Length)]
+
+DT[, outer(Sepal.Length), env = list(outer=sqrt), verbose=TRUE]
+#Argument 'j' after substitute: .Primitive("sqrt")(Sepal.Length)
+## DT[, .Primitive("sqrt")(Sepal.Length)]
+```
+And while `.Primitive("sqrt")(Sepal.Length)` still works, it is almost never the desired form.
+
+Even more importantly, if the symbol form is meant to be used, then it can, and should, be used directly in the expression, as there is no need for substitution.
+```{r substitute_fun2, result='hide'}
+DT[, sqrt(Sepal.Length)]
+```
+
+If function name to be substituted needs to be namespace-qualified then namespace and function name can be substituted as any other symbol in the expression:
+```{r substitute_fun3, result='hide', eval=FALSE}
+DT[, ns::fun(Sepal.Length), env = list(ns="base", fun="sqrt"), verbose=TRUE]
+#Argument 'j' after substitute: base::sqrt(Sepal.Length)
+## DT[, base::sqrt(Sepal.Length)]
+```
+
 ### Substitute variables and character values
 
 In the above example, we have seen a convenient feature of `substitute2`: automatic conversion from strings into names/symbols. An obvious question arises: what if we actually want to substitute a parameter with a *character* value, so as to have base R `substitute` behaviour. We provide a mechanism to escape automatic conversion by wrapping the elements into base R `I()` call. The `I` function marks an object as *AsIs*, preventing its arguments from character-to-symbol automatic conversion. (Read the `?AsIs` documentation for more details.) If base R behaviour is desired for the whole `env` argument, then it's best to wrap the whole argument in `I()`. Alternatively, each list element can be wrapped in `I()` individually. Let's explore both cases below.