Merge branch 'master' into hindi_s

Author: amansingh1011

.ci/.lintr.R

@@ -9,10 +9,10 @@ linters = c(dt_linters, all_linters(
   packages = "lintr", # TODO(lintr->3.2.0): Remove this.
   # eq_assignment_linter(),
   brace_linter(allow_single_line = TRUE),
+  implicit_integer_linter(allow_colon = TRUE),
   # TODO(michaelchirico): Activate these incrementally. These are the
   #   parameterizations that match our style guide.
   # implicit_assignment_linter(allow_lazy = TRUE, allow_scoped = TRUE),
-  # implicit_integer_linter(allow_colon = TRUE),
   # system_time_linter = undesirable_function_linter(c(
   #   system.time = "Only run timings in benchmark.Rraw"
   # )),
@@ -85,6 +85,7 @@ exclusions = c(local({
       undesirable_function_linter = Inf
     )),
     exclusion_for_dir(c("vignettes", "vignettes/fr"), list(
+      implicit_integer_linter = Inf,
       quotes_linter = Inf,
       sample_int_linter = Inf
       # strings_as_factors_linter = Inf

.ci/atime/tests.R

@@ -231,7 +231,20 @@ test.list <- atime::atime_test_list(
     },
     expr = data.table:::melt(DT, measure.vars = measure.vars),
     Slow = "fd24a3105953f7785ea7414678ed8e04524e6955", # Parent of the merge commit (https://github.com/Rdatatable/data.table/commit/ed72e398df76a0fcfd134a4ad92356690e4210ea) of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
-    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
+    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue  # Test case created directly using the atime code below (not adapted from any other benchmark), based on the issue/fix PR https://github.com/Rdatatable/data.table/pull/5054#issue-930603663 "melt should be more efficient when there are missing input columns."
+
+  # Test case created from @tdhock's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2327396833, in turn adapted from @philippechataignon's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2326714012
+  "fwrite refactored in #6393" = atime::atime_test(
+    setup = {
+      set.seed(1)
+      NC = 10L
+      L <- data.table(i=1:N)
+      L[, paste0("V", 1:NC) := replicate(NC, rnorm(N), simplify=FALSE)]
+      out.csv <- tempfile()
+    },
+    expr = data.table::fwrite(L, out.csv, compress="gzip"),
+    Before = "f339aa64c426a9cd7cf2fcb13d91fc4ed353cd31", # Parent of the first commit https://github.com/Rdatatable/data.table/commit/fcc10d73a20837d0f1ad3278ee9168473afa5ff1 in the PR https://github.com/Rdatatable/data.table/pull/6393/commits with major change to fwrite with gzip.
+    PR = "3630413ae493a5a61b06c50e80d166924d2ef89a"), # Close-to-last merge commit in the PR.
 
   tests=extra.test.list)
 # nolint end: undesirable_operator_linter.

.ci/linters/c/alloc_linter.R

@@ -4,8 +4,9 @@
 #   2. Check the next line for a check like 'if (!x || !y)'
 alloc_linter = function(c_obj) {
   lines = c_obj$lines
-  # Be a bit more precise to avoid mentions in comments
-  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(]}", lines)
+  # Be a bit more precise to avoid mentions in comments, and allow
+  #   malloc(0) to be used for convenience (e.g. #6757)
+  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(][^0]}", lines)
   if (!length(alloc_lines)) return()
   # int *tmp=(int*)malloc(...); or just int tmp=malloc(...);
   alloc_keys = lines[alloc_lines] |>

.ci/linters/c/cocci_linter.R

@@ -0,0 +1,20 @@
+cocci_linter = if (!nzchar(Sys.which("spatch"))) function(...) {} else function(c_obj) {
+  bad <- FALSE
+  for (spfile in list.files(".ci/linters/cocci", full.names = TRUE)) {
+    # Coccinelle parser gets confused sometimes, so ignore stderr and the exit code
+    out = suppressWarnings(system2(
+      "spatch",
+      shQuote(c(
+        "--sp-file", spfile, c_obj$path, "--recursive-includes",
+        "-I", R.home("include"), "-I", "src"
+      )),
+      stdout = TRUE, stderr = FALSE
+    ))
+    if (length(out) > 0) {
+      cat(sprintf("In file '%s', Coccinelle patch '%s' recommends the following changes:\n", c_obj$path, spfile))
+      writeLines(out)
+      bad <- TRUE
+    }
+  }
+  if (bad) stop("Please apply the changes above or fix the linter")
+}

.ci/linters/cocci/redundant_cast_to_same_pointer_type.cocci

@@ -0,0 +1,6 @@
+@@
+type T;
+T* E;
+@@
+- (T*)
+  E

.ci/linters/md/heading_id_linter.R

@@ -0,0 +1,26 @@
+any_mismatch = FALSE
+
+# ensure that ids are limited to alphanumerics and dashes
+# (in particular, dots and underscores break the links)
+check_header_ids = function(md) {
+  # A bit surprisingly, some headings don't start with a letter.
+  # We're interested in those that set an id to link to, i.e., end with {#id}.
+  heading_captures = regmatches(md, regexec("^#+ \\S.*[{]#([^}]*)[}]$", md))
+  lines_with_id = which(lengths(heading_captures) > 0)
+  ids = vapply(heading_captures[lines_with_id], `[`, '', 2)
+  # ids must start with a letter and consist of alphanumerics or dashes.
+  good_ids = grepl('^[A-Za-z][A-Za-z0-9-]*$', ids)
+  for (line in lines_with_id[!good_ids]) cat(sprintf(
+    "On line %d, bad heading id '%s':\n%s\n",
+    line, heading_captures[[line]][2], heading_captures[[line]][1]
+  ))
+  !all(good_ids)
+}
+
+any_error = FALSE
+for (vignette in list.files('vignettes', pattern = "[.]Rmd$", recursive = TRUE, full.name = TRUE)) {
+  cat(sprintf("Checking vignette file %s...\n", vignette))
+  rmd_lines = readLines(vignette)
+  any_error = check_header_ids(rmd_lines) || any_error
+}
+if (any_error) stop("Please fix the vignette issues above.")

.dev/CRAN_Release.cmd

@@ -3,17 +3,19 @@
 ###############################################
 
 # 1) Update messages for new release
-## (a) Update C template file: src/data.table.pot
-##     ideally, we are including _() wrapping in
-##     new PRs throughout dev cycle, and this step
-##     becomes about tying up loose ends
-## Check the output here for translatable messages
-xgettext -o /dev/stdout ./*.c \
-  --keyword=Rprintf --keyword=error --keyword=warning --keyword=STOP --keyword=DTWARN --keyword=Error --keyword=DTPRINT --keyword=snprintf:3
-
-## (b) Update R template file: src/R-data.table.pot
-##  NB: this relies on R >= 4.0 to remove a bug in update_pkg_po
-Rscript -e "tools::update_pkg_po('.')"
+dt_custom_translators = list(
+  R = 'catf:fmt|1',
+  # TODO(MichaelChirico/potools#318): restore snprintf:3 here too
+  src = c('STOP:1', 'DTWARN:1', 'DTPRINT:1')
+)
+message_db =
+  potools::get_message_data(custom_translation_functions = dt_custom_translators)
+potools::check_cracked_messages(message_db)
+potools::check_untranslated_cat(message_db)
+potools::check_untranslated_src(message_db)
+
+## (b) Update R template files (po/*.pot)
+potools::po_extract(custom_translation_functions = dt_custom_translators)
 
 # 2) Open a PR with the new templates & contact the translators
 #   * zh_CN: @hongyuanjia
@@ -114,12 +116,6 @@ grep -P "\t" ./src/*.c
 grep -n "[^A-Za-z0-9]T[^A-Za-z0-9]" ./inst/tests/tests.Rraw
 grep -n "[^A-Za-z0-9]F[^A-Za-z0-9]" ./inst/tests/tests.Rraw
 
-# All integers internally should have L suffix to avoid lots of one-item coercions
-# Where 0 numeric is intended we should perhaps use 0.0 for clarity and make the grep easier
-# 1) tolerance=0 usages in setops.R are valid numeric 0, as are anything in strings
-# 2) leave the rollends default using roll>=0 though; comments in PR #3803
-grep -Enr "^[^#]*(?:\[|==|>|<|>=|<=|,|\(|\+)\s*[-]?[0-9]+[^0-9L:.e]" R | grep -Ev "stop|warning|tolerance"
-
 # Never use ifelse. fifelse for vectors when necessary (nothing yet)
 grep -Enr "\bifelse" R
 
@@ -135,10 +131,6 @@ grep -Fn "tryCatch" ./inst/tests/*.Rraw
 # All % in *.Rd should be escaped otherwise text gets silently chopped
 grep -n "[^\]%" ./man/*.Rd
 
-# if (a & b) is either invalid or inefficient (ditto for replace & with |);
-#   if(any(a [&|] b)) is appropriate b/c of collapsing the logical vector to scalar
-grep -nr "^[^#]*if[^&#]*[^&#\"][&][^&]" R | grep -Ev "if\s*[(](?:any|all)"
-
 # seal leak potential where two unprotected API calls are passed to the same
 # function call, usually involving install() or mkChar()
 # Greppable thanks to single lines and wide screens

.devcontainer/r-devel-alpine/Dockerfile

@@ -0,0 +1,22 @@
+FROM docker.io/rhub/r-minimal:devel
+
+RUN apk update \
+  && apk add --no-cache \
+     gcc git musl-dev openmp pkgconf tzdata zlib-dev \
+  && echo 'options("repos"="https://cloud.r-project.org")' >> /usr/local/lib/R/etc/Rprofile.site
+
+ENV TZDIR=/usr/share/zoneinfo
+
+COPY DESCRIPTION .
+
+RUN Rscript -e '                                          \ 
+read.dcf("DESCRIPTION", c("Imports", "Suggests")) |>      \
+  tools:::.split_dependencies() |>                        \
+  names() |>                                              \
+  setdiff(tools:::.get_standard_package_names()$base) |>  \
+  install.packages(repos="https://cloud.r-project.org")   \
+'
+
+# setup cc()
+WORKDIR /root
+COPY .devcontainer/.Rprofile .

.devcontainer/r-devel-alpine/devcontainer.json

@@ -0,0 +1,9 @@
+{
+    "build": { "dockerfile": "Dockerfile", "context": "../.." },
+    "customizations": { "vscode": {
+       "extensions": [
+        "REditorSupport.r",
+        "ms-vscode.cpptools-extension-pack"
+       ]
+    }}
+  }

.github/ISSUE_TEMPLATE/issue_template.md

@@ -5,13 +5,18 @@ about: Report a bug or describe a new requested feature
 
 Click preview tab ^^^ above!
 
-By continuing to file this new issue / feature request, I confirm I have :
+By continuing to file this new issue / feature request, I confirm I have:
 1. searched the [live NEWS file](https://github.com/Rdatatable/data.table/blob/master/NEWS.md) to see if it has been fixed in dev already. If so, I tried the [latest dev version](https://github.com/Rdatatable/data.table/wiki/Installation#windows).
 2. looked at the titles of all the issues in the [current milestones](https://github.com/Rdatatable/data.table/milestones) and am aware of those. (Adding new information to existing issues is very helpful and appreciated.)
 3. [searched all issues](https://github.com/Rdatatable/data.table/issues) (i.e. not in a milestone yet) for similar issues to mine and will include links to them explaining why mine is different.
 4. searched on [Stack Overflow's data.table tag](http://stackoverflow.com/questions/tagged/data.table) and there is nothing similar there.
 5. read the [Support](https://github.com/Rdatatable/data.table/wiki/Support) and [Contributing](https://github.com/Rdatatable/data.table/blob/master/.github/CONTRIBUTING.md) guides.
-6. please don't tag your issue with text in the title; project members will add the appropriate tags later.
+
+Some general advice on the title and description fields for your PR
+
+- Please don't tag your issue with text in the title like '[Joins]'; project members will add the appropriate tags later.
+- Don't write text like 'Closes #xxx' in the PR title either; GitHub does not recognize this text, whereas GitHub auto-links issues in the description, [see docs](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword).
+- Title and Description fields should try and be self-contained as much as possible. The title answers "what is this change" and the description provides necessary details/thought processes/things tried but abandoned. Imagine visiting your PR in 5 years' time and trying to glean what it's all about quickly and without needing to open 10 new tabs.
 
 #### Thanks! Please remove the text above and include the two items below.