Merge branch 'master' into litedown

Author: ben-schwen

.Rbuildignore

@@ -1,4 +1,5 @@
 .dir-locals.el
+.check.translations.R
 ^\.Rprofile$
 ^data\.table_.*\.tar\.gz$
 ^config\.log$

.ci/.lintr.R

@@ -9,10 +9,10 @@ linters = c(dt_linters, all_linters(
   packages = "lintr", # TODO(lintr->3.2.0): Remove this.
   # eq_assignment_linter(),
   brace_linter(allow_single_line = TRUE),
+  implicit_integer_linter(allow_colon = TRUE),
   # TODO(michaelchirico): Activate these incrementally. These are the
   #   parameterizations that match our style guide.
   # implicit_assignment_linter(allow_lazy = TRUE, allow_scoped = TRUE),
-  # implicit_integer_linter(allow_colon = TRUE),
   # system_time_linter = undesirable_function_linter(c(
   #   system.time = "Only run timings in benchmark.Rraw"
   # )),
@@ -26,7 +26,8 @@ linters = c(dt_linters, all_linters(
   #   setwd = NULL
   # )),
   undesirable_operator_linter(),
-  # TODO(lintr#2441): Use upstream implementation.
+  # TODO(lintr#2765): Use upstream implementation.
+  # assignment_linter(operator = "="),
   assignment_linter = NULL,
   absolute_path_linter = NULL, # too many false positives
   # TODO(lintr#2442): Use this once x[ , j, by] is supported.
@@ -70,43 +71,39 @@ linters = c(dt_linters, all_linters(
 ))
 rm(dt_linters)
 
-# TODO(lintr#2172): Glob with lintr itself.
-exclusions = c(local({
-  exclusion_for_dir <- function(dir, exclusions) {
-    files = file.path("..", list.files(dir, pattern = "\\.(R|Rmd|Rraw)$", full.names=TRUE))
-    stats::setNames(rep(list(exclusions), length(files)), files)
-  }
-  c(
-    exclusion_for_dir("tests", list(
-      quotes_linter = Inf,
-      # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
-      implicit_integer_linter = Inf,
-      infix_spaces_linter = Inf,
-      undesirable_function_linter = Inf
-    )),
-    exclusion_for_dir("vignettes", list(
-      quotes_linter = Inf,
-      sample_int_linter = Inf
-      # strings_as_factors_linter = Inf
-      # system_time_linter = Inf
-    )),
-    exclusion_for_dir("inst/tests", list(
-      library_call_linter = Inf,
-      numeric_leading_zero_linter = Inf,
-      undesirable_operator_linter = Inf, # For ':::', possibly we could be more careful to only exclude ':::'.
-      # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
-      comparison_negation_linter = Inf,
-      condition_call_linter = Inf,
-      duplicate_argument_linter = Inf,
-      equals_na_linter = Inf,
-      missing_argument_linter = Inf,
-      paste_linter = Inf,
-      rep_len_linter = Inf,
-      sample_int_linter = Inf,
-      seq_linter = Inf,
-      unnecessary_lambda_linter = Inf
-    ))
+exclusions = list(
+  `../tests` = list(
+    quotes_linter = Inf,
+    # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
+    implicit_integer_linter = Inf,
+    infix_spaces_linter = Inf,
+    undesirable_function_linter = Inf
+  ),
+  `../vignettes*` = list(
+    # assignment_linter = Inf,
+    implicit_integer_linter = Inf,
+    quotes_linter = Inf,
+    sample_int_linter = Inf
+    # strings_as_factors_linter = Inf
+    # system_time_linter = Inf
+  ),
+  `../inst/tests` = list(
+    library_call_linter = Inf,
+    numeric_leading_zero_linter = Inf,
+    undesirable_operator_linter = Inf, # For ':::', possibly we could be more careful to only exclude ':::'.
+    # TODO(michaelchirico): Enforce these and re-activate them one-by-one.
+    comparison_negation_linter = Inf,
+    condition_call_linter = Inf,
+    duplicate_argument_linter = Inf,
+    equals_na_linter = Inf,
+    missing_argument_linter = Inf,
+    paste_linter = Inf,
+    rep_len_linter = Inf,
+    sample_int_linter = Inf,
+    seq_linter = Inf,
+    unnecessary_lambda_linter = Inf
+  ),
+  `../inst/tests/froll.Rraw` = list(
+    dt_test_literal_linter = Inf # TODO(michaelchirico): Fix these once #5898, #5692, #5682, #5576, #5575, #5441 are merged.
   )
-}),
-  list(`../inst/tests/froll.Rraw` = list(dt_test_literal_linter = Inf)) # TODO(michaelchirico): Fix these once #5898, #5692, #5682, #5576, #5575, #5441 are merged.
 )

.ci/README.md

@@ -1,6 +1,6 @@
 # data.table continuous integration and deployment
 
-On each Pull Request opened in GitHub we run GitHub Actions test jobs to provide prompt feedback about the status of PR. Our main CI pipeline runs on GitLab CI nightly. GitLab repository automatically mirrors our GitHub repository and runs pipeline on `master` branch every night. It tests more environments and different configurations. It publish variety of artifacts.
+On each Pull Request opened in GitHub we run GitHub Actions test jobs to provide prompt feedback about the status of PR. Our more thorough main CI pipeline runs nightly on GitLab CI. GitLab repository automatically mirrors our GitHub repository and runs pipeline on `master` branch every night. It tests more environments and different configurations. It publishes a variety of artifacts such as our [homepage](https://rdatatable.gitlab.io/data.table/) and [CRAN-like website for dev version](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html), including windows binaries for the dev version.
 
 ## Environments
 
@@ -44,3 +44,22 @@ Base R implemented helper script, [originally proposed to base R](https://svn.r-
 ### [`publish.R`](./publish.R)
 
 Base R implemented helper script to orchestrate generation of most artifacts and to arrange them nicely. It is being used only in [_integration_ stage in GitLab CI pipeline](./../.gitlab-ci.yml).
+
+## GitLab Open Source Program
+
+We are currently part of the [GitLab for Open Source Program](https://about.gitlab.com/solutions/open-source/). This gives us 50,000 compute minutes per month for our GitLab CI. Our license needs to be renewed yearly (around July) and is currently managed by @ben-schwen.
+
+## Updating CI pipeline
+
+Basic CI checks are also run on every push to the GitLab repository. This can **and should** be used for PRs changing the CI pipeline before merging them to master.
+
+```shell
+# fetch changes from remote (GitHub) and push them to GitLab 
+git fetch [email protected]:Rdatatable/data.table.git new_branch:new_branch
+git push
+# after updating on GitHub, pull changes from remote and push to GitLab
+git pull [email protected]:Rdatatable/data.table.git new_branch
+git push
+```
+
+Make sure to include a link to the pipeline results in your PR. 

.ci/atime/tests.R

@@ -231,7 +231,20 @@ test.list <- atime::atime_test_list(
     },
     expr = data.table:::melt(DT, measure.vars = measure.vars),
     Slow = "fd24a3105953f7785ea7414678ed8e04524e6955", # Parent of the merge commit (https://github.com/Rdatatable/data.table/commit/ed72e398df76a0fcfd134a4ad92356690e4210ea) of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
-    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
+    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue  # Test case created directly using the atime code below (not adapted from any other benchmark), based on the issue/fix PR https://github.com/Rdatatable/data.table/pull/5054#issue-930603663 "melt should be more efficient when there are missing input columns."
+
+  # Test case created from @tdhock's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2327396833, in turn adapted from @philippechataignon's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2326714012
+  "fwrite refactored in #6393" = atime::atime_test(
+    setup = {
+      set.seed(1)
+      NC = 10L
+      L <- data.table(i=1:N)
+      L[, paste0("V", 1:NC) := replicate(NC, rnorm(N), simplify=FALSE)]
+      out.csv <- tempfile()
+    },
+    expr = data.table::fwrite(L, out.csv, compress="gzip"),
+    Before = "f339aa64c426a9cd7cf2fcb13d91fc4ed353cd31", # Parent of the first commit https://github.com/Rdatatable/data.table/commit/fcc10d73a20837d0f1ad3278ee9168473afa5ff1 in the PR https://github.com/Rdatatable/data.table/pull/6393/commits with major change to fwrite with gzip.
+    PR = "3630413ae493a5a61b06c50e80d166924d2ef89a"), # Close-to-last merge commit in the PR.
 
   tests=extra.test.list)
 # nolint end: undesirable_operator_linter.

.ci/ci.R

@@ -111,7 +111,7 @@ mirror.packages <-
 function(pkgs,
          which = c("Depends", "Imports", "LinkingTo"),
          repos = getOption("repos"),
-         type = c("source", "mac.binary", "win.binary"),
+         type = c("source", "mac.binary.big-sur-arm64", "win.binary"),
          repodir,
          except.repodir = repodir,
          except.priority = "base",
@@ -169,7 +169,8 @@ function(pkgs,
     newpkgs <- newpkgs[availpkgs]
   }
 
-  pkgsext <- switch(type,
+  typeshort <- if (startsWith(type, "mac.binary.")) "mac.binary" else type
+  pkgsext <- switch(typeshort,
                     "source" = "tar.gz",
                     "mac.binary" = "tgz",
                     "win.binary" = "zip")
@@ -181,7 +182,7 @@ function(pkgs,
   dp <- utils::download.packages(pkgs = newpkgs, destdir = destdir,
                                  available = db, contriburl = repos.url,
                                  type = type, method = method, quiet = quiet)
-  tools::write_PACKAGES(dir = destdir, type = type, ...)
+  tools::write_PACKAGES(dir = destdir, type = typeshort, ...)
   dp
 }
 

.ci/linters/c/alloc_linter.R

@@ -4,8 +4,9 @@
 #   2. Check the next line for a check like 'if (!x || !y)'
 alloc_linter = function(c_obj) {
   lines = c_obj$lines
-  # Be a bit more precise to avoid mentions in comments
-  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(]}", lines)
+  # Be a bit more precise to avoid mentions in comments, and allow
+  #   malloc(0) to be used for convenience (e.g. #6757)
+  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(][^0]}", lines)
   if (!length(alloc_lines)) return()
   # int *tmp=(int*)malloc(...); or just int tmp=malloc(...);
   alloc_keys = lines[alloc_lines] |>

.ci/linters/c/cocci_linter.R

@@ -0,0 +1,20 @@
+cocci_linter = if (!nzchar(Sys.which("spatch"))) function(...) {} else function(c_obj) {
+  bad <- FALSE
+  for (spfile in list.files(".ci/linters/cocci", full.names = TRUE)) {
+    # Coccinelle parser gets confused sometimes, so ignore stderr and the exit code
+    out = suppressWarnings(system2(
+      "spatch",
+      shQuote(c(
+        "--sp-file", spfile, c_obj$path, "--recursive-includes",
+        "-I", R.home("include"), "-I", "src"
+      )),
+      stdout = TRUE, stderr = FALSE
+    ))
+    if (length(out) > 0) {
+      cat(sprintf("In file '%s', Coccinelle patch '%s' recommends the following changes:\n", c_obj$path, spfile))
+      writeLines(out)
+      bad <- TRUE
+    }
+  }
+  if (bad) stop("Please apply the changes above or fix the linter")
+}

.ci/linters/cocci/redundant_cast_to_same_pointer_type.cocci

@@ -0,0 +1,6 @@
+@@
+type T;
+T* E;
+@@
+- (T*)
+  E

.ci/linters/md/heading_id_linter.R

@@ -0,0 +1,26 @@
+any_mismatch = FALSE
+
+# ensure that ids are limited to alphanumerics and dashes
+# (in particular, dots and underscores break the links)
+check_header_ids = function(md) {
+  # A bit surprisingly, some headings don't start with a letter.
+  # We're interested in those that set an id to link to, i.e., end with {#id}.
+  heading_captures = regmatches(md, regexec("^#+ \\S.*[{]#([^}]*)[}]$", md))
+  lines_with_id = which(lengths(heading_captures) > 0)
+  ids = vapply(heading_captures[lines_with_id], `[`, '', 2)
+  # ids must start with a letter and consist of alphanumerics or dashes.
+  good_ids = grepl('^[A-Za-z][A-Za-z0-9-]*$', ids)
+  for (line in lines_with_id[!good_ids]) cat(sprintf(
+    "On line %d, bad heading id '%s':\n%s\n",
+    line, heading_captures[[line]][2], heading_captures[[line]][1]
+  ))
+  !all(good_ids)
+}
+
+any_error = FALSE
+for (vignette in list.files('vignettes', pattern = "[.]Rmd$", recursive = TRUE, full.name = TRUE)) {
+  cat(sprintf("Checking vignette file %s...\n", vignette))
+  rmd_lines = readLines(vignette)
+  any_error = check_header_ids(rmd_lines) || any_error
+}
+if (any_error) stop("Please fix the vignette issues above.")

.ci/publish.R

@@ -27,7 +27,7 @@ format.bins <- function(ver, bin_ver, cran.home, os.type, pkg, version, repodir)
     plat.path = "windows"
   } else if (os.type=="macosx") {
     ext = "tgz"
-    plat.path = "macosx/el-capitan"
+    plat.path = "macosx/big-sur-arm64"
   } else stop("format.bins only valid for 'windows' or 'macosx' os.type")
   file = sprintf("bin/%s/contrib/%s/%s_%s.%s", plat.path, bin_ver, pkg, version, ext)
   fe = file.exists(file.path(repodir, file))