Merge branch 'master' into benchmark

Author: MichaelChirico

.Rbuildignore

@@ -1,5 +1,8 @@
+.dir-locals.el
+.check.translations.R
 ^\.Rprofile$
 ^data\.table_.*\.tar\.gz$
+^config\.log$
 ^vignettes/plots/figures$
 ^\.Renviron$
 ^[^/]+\.R$
@@ -11,18 +14,22 @@
 
 ^\.ci$
 ^\.dev$
+^\.devcontainer$
 ^\.graphics$
 ^\.github$
+^\.vscode$
+^\.zed$
 
-^\.travis\.yml$
-^\.appveyor\.yml$
 ^\.gitlab-ci\.yml$
 
 ^Makefile$
 ^NEWS\.0\.md$
-^README\.md$
+^NEWS\.1\.md$
 ^_pkgdown\.yml$
 ^src/Makevars$
+^CODEOWNERS$
+^GOVERNANCE\.md$
+^Seal_of_Approval\.md$
 
 ^\.RData$
 ^\.Rhistory$
@@ -32,10 +39,16 @@
 ^.*\.Rproj$
 ^\.Rproj\.user$
 ^\.idea$
+^\.libs$
 
 ^.*\.dll$
 
 ^bus$
 ^pkgdown$
+^docs$
 ^lib$
 ^library$
+^devwd$
+
+# only the inst/po compressed files are needed, not raw .pot/.po
+^po$

.Rprofile

@@ -0,0 +1,117 @@
+dt_linters = new.env()
+for (f in list.files('.ci/linters/r', full.names=TRUE)) sys.source(f, dt_linters)
+rm(f)
+
+# NB: Could do this inside the linter definition, this separation makes those files more standardized
+dt_linters <- eapply(dt_linters, function(linter_factory) linter_factory())
+
+linters = c(dt_linters, all_linters(
+  packages = "lintr", # TODO(lintr->3.2.0): Remove this.
+  # eq_assignment_linter(),
+  brace_linter(allow_single_line = TRUE),
+  implicit_integer_linter(allow_colon = TRUE),
+  # TODO(michaelchirico): Activate these incrementally. These are the
+  #   parameterizations that match our style guide.
+  # implicit_assignment_linter(allow_lazy = TRUE, allow_scoped = TRUE),
+  # system_time_linter = undesirable_function_linter(c(
+  #   system.time = "Only run timings in benchmark.Rraw"
+  # )),
+  undesirable_function_linter(c(
+    cat = "Use catf to enable translations",
+    message = "Use messagef to avoid fragmented translations.",
+    warning = "Use warningf to avoid fragmented translations.",
+    stop = "Use stopf to avoid fragmented translations.",
+    rev = "Use frev internally, or setfrev if by-reference is safe.",
+    NULL
+  )),
+  # undesirable_function_linter(modify_defaults(
+  #   default_undesirable_functions,
+  #   ifelse = "Use fifelse instead.",
+  #   Sys.setenv = NULL,
+  #   library = NULL,
+  #   options = NULL,
+  #   par = NULL,
+  #   setwd = NULL
+  # )),
+  undesirable_operator_linter(),
+  # TODO(lintr#2765): Use upstream implementation.
+  # assignment_linter(operator = "="),
+  assignment_linter = NULL,
+  absolute_path_linter = NULL, # too many false positives
+  # TODO(lintr#2442): Use this once x[ , j, by] is supported.
+  commas_linter = NULL,
+  commented_code_linter = NULL,
+  # TODO(linter->3.2.0): Activate this.
+  consecutive_assertion_linter = NULL,
+  cyclocomp_linter = NULL,
+  function_argument_linter = NULL,
+  indentation_linter = NULL,
+  infix_spaces_linter = NULL,
+  line_length_linter = NULL,
+  missing_package_linter = NULL,
+  namespace_linter = NULL,
+  nonportable_path_linter = NULL,
+  object_name_linter = NULL,
+  object_usage_linter = NULL,
+  quotes_linter = NULL,
+  semicolon_linter = NULL,
+  spaces_inside_linter = NULL,
+  spaces_left_parentheses_linter = NULL,
+  # TODO(michaelchirico): Only exclude from vignettes, not sure what's wrong.
+  strings_as_factors_linter = NULL,
+  # TODO(lintr->3.2.0): Fix on a valid TODO style, enforce it, and re-activate.
+  todo_comment_linter = NULL,
+  # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
+  brace_linter = NULL,
+  fixed_regex_linter = NULL,
+  if_not_else_linter = NULL,
+  implicit_assignment_linter = NULL,
+  implicit_integer_linter = NULL,
+  keyword_quote_linter = NULL,
+  object_overwrite_linter = NULL,
+  paren_body_linter = NULL,
+  redundant_equals_linter = NULL,
+  undesirable_function_linter = NULL,
+  unnecessary_concatenation_linter = NULL,
+  unnecessary_nesting_linter = NULL,
+  unreachable_code_linter = NULL,
+  unused_import_linter = NULL
+))
+rm(dt_linters)
+
+exclusions = list(
+  `../tests` = list(
+    quotes_linter = Inf,
+    # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
+    implicit_integer_linter = Inf,
+    infix_spaces_linter = Inf,
+    undesirable_function_linter = Inf
+  ),
+  `../vignettes*` = list(
+    # assignment_linter = Inf,
+    implicit_integer_linter = Inf,
+    quotes_linter = Inf,
+    sample_int_linter = Inf
+    # strings_as_factors_linter = Inf
+    # system_time_linter = Inf
+  ),
+  `../inst/tests` = list(
+    library_call_linter = Inf,
+    numeric_leading_zero_linter = Inf,
+    undesirable_operator_linter = Inf, # For ':::', possibly we could be more careful to only exclude ':::'.
+    # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
+    comparison_negation_linter = Inf,
+    condition_call_linter = Inf,
+    duplicate_argument_linter = Inf,
+    equals_na_linter = Inf,
+    missing_argument_linter = Inf,
+    paste_linter = Inf,
+    rep_len_linter = Inf,
+    sample_int_linter = Inf,
+    seq_linter = Inf,
+    unnecessary_lambda_linter = Inf
+  ),
+  `../inst/tests/froll.Rraw` = list(
+    dt_test_literal_linter = Inf # TODO(michaelchirico): Fix these once #5898, #5692, #5682, #5576, #5575, #5441 are merged.
+  )
+)

.appveyor.yml

@@ -1,71 +1,87 @@
 # data.table continuous integration and deployment
 
-On each Pull Request opened in GitHub we run Travis CI and Appveyor to provide prompt feedback about the status of PR. Our main CI pipeline runs on GitLab CI. GitLab repository automatically mirrors our GitHub repository and runs pipeline on `master` branch. It tests more environments and different configurations. It publish variety of artifacts.
+On each Pull Request opened in GitHub we run GitHub Actions test jobs to provide prompt feedback about the status of PR. Our more thorough main CI pipeline runs nightly on GitLab CI. In addition to branches pushed directly, the GitLab repository automatically mirrors our GitHub repository and runs pipeline on the `master` branch every night. It tests more environments and different configurations. It publishes a variety of artifacts such as our [homepage](https://rdatatable.gitlab.io/data.table/) and [CRAN-like website for dev version](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html), including windows binaries for the dev version.
 
 ## Environments
 
 ### [GitLab CI](./../.gitlab-ci.yml)
 
 Test jobs:
-- `test-rel-lin` - `r-release` on Linux, most comprehensive test environment, `-O3 -flto -fno-common -Wunused-result`, extra check for no compilation warnings, includes testing [_with other packages_](./../inst/tests/other.Rraw) ([extended suggests](./../inst/tests/tests-DESCRIPTION))
-- `test-rel-cran-lin` - `--as-cran` on Linux, `-g0`, extra check for final status of `R CMD check` where we allow one NOTE (_size of tarball_).
-- `test-dev-cran-lin` - `r-devel` and `--as-cran` on Linux, `--with-recommended-packages --enable-strict-barrier --disable-long-double`, tests for compilation warnings in pkg install and new NOTEs/Warnings in pkg check, and because it is R-devel it is marked as allow_failure
-- `test-rel-vanilla-lin` - `r-release` on Linux, no suggested deps, no OpenMP, `-O0`, tracks memory usage during tests
-- `test-310-cran-lin` - R 3.1.0 on Linux
-- `test-344-cran-lin` - R 3.4.4 on Linux
-- `test-350-cran-lin` - R 3.5.0 on Linux, no `r-recommended`
-- `test-rel-win` - `r-release` on Windows
-- `test-dev-win` - `r-devel` on Windows
-- `test-rel-osx` - MacOSX build not yet deployed, see [#3326](https://github.com/Rdatatable/data.table/issues/3326) for status
+- `test-lin-rel` - `r-release` on Linux, most comprehensive test environment, force all suggests, `-O3 -flto=auto -fno-common -Wunused-result`, test for no compilation warnings.
+- `test-lin-rel-vanilla` - `r-release` on Linux, no suggested deps, no zlib, no OpenMP, flags `-g -O0 -fno-openmp`, skip manual and vignettes.
+- `test-lin-rel-cran` - `--as-cran` on Linux, strict test for final status of `R CMD check`.
+- `test-lin-dev-gcc-strict-cran` - `--as-cran` on Linux, `r-devel` built with `-enable-strict-barrier --disable-long-double`, test for compilation warnings, test for new NOTEs/WARNINGs from `R CMD check`.
+- `test-lin-dev-clang-cran` - same as `gcc-strict` job but R built with `clang` and  no `--enable-strict-barrier --disable-long-double` flags.
+- `test-lin-ancient-cran` - Stated R dependency version (currently 3.4.0) on Linux.
+- `test-lin-dev-clang-san` - `r-devel` on Linux built with `clang -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
+- `test-lin-dev-gcc-san` - `r-devel` on Linux built with `gcc -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
+- `test-win-rel` - `r-release` on Windows.
+- `test-win-dev` - `r-devel` on Windows.
+- `test-win-old` - `r-oldrel` on Windows.
+- `test-mac-rel` - `r-release` on macOS.
+- `test-mac-old` - `r-oldrel` on macOS.
+
+The CI steps for the tests are [required](https://github.com/Rdatatable/data.table/blob/55eb0f160b169398d51f138131c14a66c86e5dc9/.ci/publish.R#L162-L168) to be named according to the pattern `test-(lin|win|mac)-<R version>[-<suffix>]*`, where `<R version>` is `rel`, `dev`, `old`, `ancient`, or three digits comprising an R version (e.g. `362` corresponding to R-3.6.2).
+
+Tests jobs are allowed to fail, summary and logs of test jobs are later published at _CRAN-like checks_ page, see artifacts below.
 
 Artifacts:
 - [homepage](https://rdatatable.gitlab.io/data.table) - made with [pkgdown](https://github.com/r-lib/pkgdown)
 - [html manual](https://rdatatable.gitlab.io/data.table/library/data.table/html/00Index.html)
 - [pdf manual](https://rdatatable.gitlab.io/data.table/web/packages/data.table/data.table.pdf)
 - [html vignettes](https://rdatatable.gitlab.io/data.table/library/data.table/doc/index.html)
-- R packages repository for `data.table` and all _Suggests_ dependencies, url: `https://Rdatatable.gitlab.io/data.table`
+- R packages repository for `data.table` and all _Suggests_ dependencies, url: `https://rdatatable.gitlab.io/data.table`
   - sources
-  - Windows binaries for `r-release` and `r-devel`
+  - Windows binaries for `r-release`, `r-devel` and `r-oldrel`
 - [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
-- [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html) - note that all artifacts, including check results page, are being published only when all test jobs successfully pass, thus one will not see an _ERROR_ status there (unless error happened on a job marked as `allow_failure`).
-- [docker images](https://gitlab.com/Rdatatable/data.table/container_registry) - copy/paste-able `docker pull` commands can be found at the bottom of our [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
+- [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html)
 
-### [Travis CI](./../.travis.yml)
+### [GitHub Actions](./../.github/workflows)
 
-Test jobs:
-- `r-release` on Linux, includes code coverage check
-- _(might be disabled)_ `r-release` on OSX
+TODO document
 
-Artifacts:
-- R packages repository having `data.table` sources only, url: `https://Rdatatable.github.io/data.table`
-- code coverage stats pushed to [codecov.io/gh/Rdatatable/data.table](https://codecov.io/gh/Rdatatable/data.table)
+## CI tools
 
-### [Appveyor](./../.appveyor.yml)
+### [`ci.R`](./ci.R)
 
-Test jobs:
-- Windows `r-release`
-- _(might be disabled)_ Windows `r-devel`
+Base R implemented helper script, [originally proposed to base R](https://svn.r-project.org/R/branches/tools4pkgs/src/library/tools/R/packages.R), that ease the process of extracting dependency information from description files, and to mirror packages and their recursive dependencies from CRAN to local CRAN-like directory. It is used in [GitLab CI pipeline](./../.gitlab-ci.yml).
 
-Artifacts:
-- Windows `r-release` binaries accessed only via web UI
+### [`publish.R`](./publish.R)
 
-## Tools
+Base R implemented helper script to orchestrate generation of most artifacts and to arrange them nicely. It is being used only in [_integration_ stage in GitLab CI pipeline](./../.gitlab-ci.yml).
 
-### [`ci.R`](./ci.R)
+### [`lint.R`](./lint.R)
 
-Base R implemented helper script, [originally proposed to R](https://svn.r-project.org/R/branches/tools4pkgs/src/library/tools/R/packages.R), that ease the process of extracting dependency information from description files, also to mirror packages and their recursive dependencies from CRAN to local CRAN-like directory. It is widely used in our [GitLab CI pipeline](./../.gitlab-ci.yml).
+Base R runner for the manual (non-`lintr`) lint checks to be run from GitHub Actions during the code quality check. The command line arguments are as follows:
+1. Path to the directory containing files defining the linters. A linter is a function that accepts one argument (typically the path to the file) and signals an error if it fails the lint check.
+2. Path to the directory containing files to check.
+3. A regular expression matching the files to check.
 
-### [`publish.R`](./publish.R)
+One of the files in the linter directory may define the `.preprocess` function, which must accept one file path and return a value that other linter functions will understand. The function may also return `NULL` to indicate that the file must be skipped.
+
+Example command lines:
+
+```sh
+Rscript .ci/lint.R .ci/linters/c src '[.][ch]$'
+Rscript .ci/lint.R .ci/linters/po po '[.]po$'
+Rscript .ci/lint.R .ci/linters/md . '[.]R?md$'
+```
+
+## GitLab Open Source Program
 
-Base R implemented helper script to orchestrate generation of most artifacts. It is being used only in [_integration_ stage in GitLab CI pipeline](./../.gitlab-ci.yml).
+We are currently part of the [GitLab for Open Source Program](https://about.gitlab.com/solutions/open-source/). This gives us 50,000 compute minutes per month for our GitLab CI. Our license needs to be renewed yearly (around July) and is currently managed by @ben-schwen.
 
-### [`Dockerfile.in`](./Dockerfile.in)
+## Updating CI pipeline
 
-Template file to produce `Dockerfile` for, as of now, three docker images. Docker images are being built and published in [_deploy_ stage in GitLab CI pipeline](./../.gitlab-ci.yml).
-- `r-base-dev` using `r-release`: publish docker image of `data.table` on R-release
-- `r-builder` using `r-release`: publish on R-release and OS dependencies for building Rmarkdown vignettes
-- `r-devel`: publish docker image of `data.table` on R-devel built with `--with-recommended-packages --enable-strict-barrier --disable-long-double`
+Basic CI checks are also run on every push to the GitLab repository. This can **and should** be used for PRs changing the CI pipeline before merging them to master.
 
-### [`deploy.sh`](./deploy.sh)
+```shell
+# fetch changes from remote (GitHub) and push them to GitLab 
+git fetch [email protected]:Rdatatable/data.table.git new_branch:new_branch
+git push
+# after updating on GitHub, pull changes from remote and push to GitLab
+git pull [email protected]:Rdatatable/data.table.git new_branch
+git push
+```
 
-Script used on Travis CI to publish CRAN-like repository of `data.table` sources. It publishes to `gh-pages` branch in GitHub repository. It depends on a token, which is provided based on `secure` environment variable in [.travis.yml](./../.travis.yml). It has been generated by @jangorecki.
+Make sure to include a link to the pipeline results in your PR.