Merge branch 'master' into bench-vign

Author: jangorecki

.Rbuildignore

@@ -1,23 +1,44 @@
-^\.travis\.yml$
-^CONTRIBUTING\.md$
-^ISSUE_TEMPLATE\.md$
-^CRAN_Release\.cmd$
-^deploy\.sh$
+.dir-locals.el
+^\.Rprofile$
+^data\.table_.*\.tar\.gz$
+^vignettes/plots/figures$
+^\.Renviron$
+^[^/]+\.R$
+^[^/]+\.csv$
+^[^/]+\.csvy$
+^[^/]+\.RDS$
+^[^/]+\.diff$
+^[^/]+\.patch$
+
+^\.ci$
+^\.dev$
+^\.devcontainer$
+^\.graphics$
+^\.github$
+
+^\.appveyor\.yml$
+^\.gitlab-ci\.yml$
+
+^Makefile$
+^NEWS\.0\.md$
+^_pkgdown\.yml$
+^src/Makevars$
+^CODEOWNERS$
+
+^\.RData$
+^\.Rhistory$
+
 ^\.emacs\.desktop
 ^\.emacs\.desktop\.lock
 ^.*\.Rproj$
 ^\.Rproj\.user$
-^appveyor\.yml$
-^.gitlab-ci\.yml$
 ^\.idea$
-^\.RData$
-^\.Rhistory$
-^data\.table_.*\.tar\.gz$
-^cc\.R$
-^ci\.R$
-^publish\.R$
-^Makefile$
-^NEWS.0.md$
+^\.libs$
+
+^.*\.dll$
+
 ^bus$
-^Dockerfile$
-^Dockerfile\.in$
+^pkgdown$
+^lib$
+^library$
+^devwd$

appveyor.yml .appveyor.ymlappveyor.yml renamed to .appveyor.yml

@@ -14,21 +14,22 @@ skip_branch_with_pr: true
 
 environment:
   global:
-    CRAN: http://cloud.r-project.org/
+    CRAN: http://cloud.r-project.org
     WARNINGS_ARE_ERRORS: 1
-    R_CHECK_ARGS: --no-manual
-# R_CHECK_ARGS specified in order to turn off --as-cran (on by default) as that can be slow
-    R_ARCH: x64
-# multiarch is on by default which runs tests on both 32bit R and 64bit R in one x64 job; i.e. very nice and convenient for all.
-# The default for R_ARCH is i386, though, for which multiarch would just compile and test 32bit, hence setting R_ARCH to x64
-    GCC_PATH: mingw_64
-# Default GCC_PATH appears to be gcc-4.6.3 which is now unsupported as from Rtools.exe v3.4.
+    R_CHECK_ARGS: --as-cran --no-manual
+# --no-manual to avoid error 'pdflatex is not available'
+# --as-cran no longer a lot slower (now takes under 6 mins with and without); logs show _R_CHECK_CRAN_INCOMING_=FALSE which could take 5+ mins
+    _R_CHECK_NO_STOP_ON_TEST_ERROR_: true
+# continue tests even if some script failed
+    _R_CHECK_TESTS_NLINES_: 0
+# Block truncation of any error messages in R CMD check
+# R is 64-bit only on Windows from 4.2.0 (prior default was build and test both 32bit and 64bit) so we no longer use R_ARCH to pick one to reduce CI time in PRs
 
   matrix:
 
   - R_VERSION: release  # the single Windows.zip binary (both 32bit/64bit) that users following dev version of installation instructions should click
 
-  - R_VERSION: devel
+#  - R_VERSION: devel   # Never turn back on. GLCI after merge covers latest daily R-devel very well, so we shouldn't confuse and slow down PR dev cycle by measuring PRs against daily R-devel too. If a change in R-devel yesterday breaks the PR, it's very unlikely to be due to something in the PR. So we should accept the PR if it passes R-release and fix separately anything related to R-devel which we'll see from GLCI.
 
 before_build:
   - cmd: ECHO no Revision metadata added to DESCRIPTION
@@ -41,6 +42,7 @@ build_script:
   - travis-tool.sh r_install xts
   - travis-tool.sh r_install nanotime
   - travis-tool.sh r_install R.utils
+  - travis-tool.sh r_install yaml
 
 test_script:
   - travis-tool.sh run_tests

.ci/README.md

@@ -0,0 +1,50 @@
+# data.table continuous integration and deployment
+
+On each Pull Request opened in GitHub we run GitHub Actions test jobs to provide prompt feedback about the status of PR. Our main CI pipeline runs on GitLab CI nightly. GitLab repository automatically mirrors our GitHub repository and runs pipeline on `master` branch every night. It tests more environments and different configurations. It publish variety of artifacts.
+
+## Environments
+
+### [GitLab CI](./../.gitlab-ci.yml)
+
+Test jobs:
+- `test-lin-rel` - `r-release` on Linux, most comprehensive test environment, force all suggests, `-O3 -flto=auto -fno-common -Wunused-result`, test for no compilation warnings.
+- `test-lin-rel-vanilla` - `r-release` on Linux, no suggested deps, no zlib, no OpenMP, flags `-g -O0 -fno-openmp`, skip manual and vignettes.
+- `test-lin-rel-cran` - `--as-cran` on Linux, strict test for final status of `R CMD check`.
+- `test-lin-dev-gcc-strict-cran` - `--as-cran` on Linux, `r-devel` built with `-enable-strict-barrier --disable-long-double`, test for compilation warnings, test for new NOTEs/WARNINGs from `R CMD check`.
+- `test-lin-dev-clang-cran` - same as `gcc-strict` job but R built with `clang` and  no `--enable-strict-barrier --disable-long-double` flags.
+- `test-lin-310-cran` - R 3.1.0 on Linux, stated R dependency version.
+- `test-win-rel` - `r-release` on Windows.
+- `test-win-dev` - `r-devel` on Windows.
+- `test-win-old` - `r-oldrel` on Windows.
+- `test-mac-rel` - macOS build not yet available, see [#3326](https://github.com/Rdatatable/data.table/issues/3326) for status
+
+Tests jobs are allowed to fail, summary and logs of test jobs are later published at _CRAN-like checks_ page, see artifacts below.
+
+Artifacts:
+- [homepage](https://rdatatable.gitlab.io/data.table) - made with [pkgdown](https://github.com/r-lib/pkgdown)
+- [html manual](https://rdatatable.gitlab.io/data.table/library/data.table/html/00Index.html)
+- [pdf manual](https://rdatatable.gitlab.io/data.table/web/packages/data.table/data.table.pdf)
+- [html vignettes](https://rdatatable.gitlab.io/data.table/library/data.table/doc/index.html)
+- R packages repository for `data.table` and all _Suggests_ dependencies, url: `https://rdatatable.gitlab.io/data.table`
+  - sources
+  - Windows binaries for `r-release`, `r-devel` and `r-oldrel`
+- [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
+- [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html)
+
+### [GitHub Actions](./../.github/workflows)
+
+TODO document
+
+### [Appveyor](./../.appveyor.yml)
+
+TODO document
+
+## CI tools
+
+### [`ci.R`](./ci.R)
+
+Base R implemented helper script, [originally proposed to base R](https://svn.r-project.org/R/branches/tools4pkgs/src/library/tools/R/packages.R), that ease the process of extracting dependency information from description files, and to mirror packages and their recursive dependencies from CRAN to local CRAN-like directory. It is used in [GitLab CI pipeline](./../.gitlab-ci.yml).
+
+### [`publish.R`](./publish.R)
+
+Base R implemented helper script to orchestrate generation of most artifacts and to arrange them nicely. It is being used only in [_integration_ stage in GitLab CI pipeline](./../.gitlab-ci.yml).

ci.R .ci/ci.Rci.R renamed to .ci/ci.R

@@ -5,10 +5,10 @@
 ## added ver argument to produce R version independent urls
 ## https://bugs.r-project.org/bugzilla3/show_bug.cgi?id=17420
 contrib.url <-
-function (repos, type = getOption("pkgType"), ver) 
+function (repos, type = getOption("pkgType"), ver)
 {
   type <- utils:::resolvePkgType(type)
-  if (is.null(repos)) 
+  if (is.null(repos))
     return(NULL)
   if ("@CRAN@" %in% repos && interactive()) {
     cat(gettext("--- Please select a CRAN mirror for use in this session ---"), "\n", sep = "")
@@ -17,12 +17,12 @@ function (repos, type = getOption("pkgType"), ver)
     m <- match("@CRAN@", repos)
     nm <- names(repos)
     repos[m] <- getOption("repos")["CRAN"]
-    if (is.null(nm)) 
+    if (is.null(nm))
       nm <- rep("", length(repos))
     nm[m] <- "CRAN"
     names(repos) <- nm
   }
-  if ("@CRAN@" %in% repos) 
+  if ("@CRAN@" %in% repos)
     stop("trying to use CRAN without setting a mirror")
   if(missing(ver)) {
     ver <- paste(R.version$major, strsplit(R.version$minor, ".", fixed=TRUE)[[1L]][1L], sep = ".")
@@ -37,17 +37,18 @@ function (repos, type = getOption("pkgType"), ver)
   res <- switch(
     type,
     source = paste(gsub("/$", "", repos), "src", "contrib", sep = "/"),
-    mac.binary = paste(gsub("/$", "", repos), "bin", mac.path, "contrib", ver, sep = "/"), 
+    mac.binary = paste(gsub("/$", "", repos), "bin", mac.path, "contrib", ver, sep = "/"),
     win.binary = paste(gsub("/$", "", repos), "bin", "windows", "contrib", ver, sep = "/")
   )
   res
 }
 
 ## returns dependencies for a package based on its DESCRIPTION file
 dcf.dependencies <-
-function(file = "DESCRIPTION", 
+function(file = "DESCRIPTION",
          which = NA,
-         except.priority = "base") {
+         except.priority = "base",
+         exclude = NULL) {
   if (!is.character(file) || !length(file) || !all(file.exists(file)))
     stop("file argument must be character of filepath(s) to existing DESCRIPTION file(s)")
   if (!is.character(except.priority))
@@ -71,15 +72,21 @@ function(file = "DESCRIPTION",
   }, which = which), use.names = FALSE)
   local.extract_dependency_package_names = function (x) {
     ## do not filter out R like tools:::.extract_dependency_package_names, used for web/$pkg/index.html
-    if (is.na(x)) 
+    if (is.na(x))
       return(character())
     x <- unlist(strsplit(x, ",[[:space:]]*"))
     x <- sub("[[:space:]]*([[:alnum:].]+).*", "\\1", x)
     x[nzchar(x)]
   }
   x <- unlist(lapply(x, local.extract_dependency_package_names))
   except <- if (length(except.priority)) c("R", unlist(tools:::.get_standard_package_names()[except.priority], use.names = FALSE))
-  setdiff(x, except)
+  x = setdiff(x, except)
+  if (length(exclude)) {  # to exclude knitr/rmarkdown, 5294
+    if (!is.character(exclude) || anyDuplicated(exclude))
+      stop("exclude may be NULL or a character vector containing no duplicates")
+    x = setdiff(x, exclude)
+  }
+  x
 }
 
 ## returns additional repositories for dependency packages based on its DESCRIPTION file
@@ -101,13 +108,13 @@ function(file = "DESCRIPTION") {
 ## download dependencies recursively for provided packages
 ## put all downloaded packages into local repository
 mirror.packages <-
-function(pkgs, 
-         which = c("Depends", "Imports", "LinkingTo"), 
-         repos = getOption("repos"), 
-         type = c("source", "mac.binary", "win.binary"), 
-         repodir, 
-         except.repodir = repodir, 
-         except.priority = "base", 
+function(pkgs,
+         which = c("Depends", "Imports", "LinkingTo"),
+         repos = getOption("repos"),
+         type = c("source", "mac.binary", "win.binary"),
+         repodir,
+         except.repodir = repodir,
+         except.priority = "base",
          method,
          quiet = TRUE,
          binary.ver,
@@ -161,7 +168,7 @@ function(pkgs,
     warning(sprintf("Packages binaries could not be found in provided reposistories for R version %s: %s", binary.ver, paste(newpkgs[!availpkgs], collapse = ", ")))
     newpkgs <- newpkgs[availpkgs]
   }
-  
+
   pkgsext <- switch(type,
                     "source" = "tar.gz",
                     "mac.binary" = "tgz",
@@ -171,14 +178,10 @@ function(pkgs,
   unlink(dlfiles[file.exists(dlfiles)])
   ## repos argument is not used in download.packages, only as default for contriburl argument
   ## we provide contriburl to avoid interactive CRAN menu popup twice in mirror.packages
-  dp <- utils::download.packages(pkgs = newpkgs, destdir = destdir, 
-                                 available = db, contriburl = repos.url, 
+  dp <- utils::download.packages(pkgs = newpkgs, destdir = destdir,
+                                 available = db, contriburl = repos.url,
                                  type = type, method = method, quiet = quiet)
   tools::write_PACKAGES(dir = destdir, type = type, ...)
   dp
 }
 
-## set repositories for CI tests
-if (as.logical(Sys.getenv("GITLAB_CI","false")) && identical(Sys.getenv("CI_PROJECT_NAME"), "data.table")) {
-  options("repos" = if (.Platform$OS.type == "windows") file.path("file://",getwd(),"bus/mirror-packages/cran") else file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)))
-}