Merge branch 'master' into automate-lang-links

Author: aitap

.ci/.lintr.R

@@ -9,10 +9,10 @@ linters = c(dt_linters, all_linters(
   packages = "lintr", # TODO(lintr->3.2.0): Remove this.
   # eq_assignment_linter(),
   brace_linter(allow_single_line = TRUE),
+  implicit_integer_linter(allow_colon = TRUE),
   # TODO(michaelchirico): Activate these incrementally. These are the
   #   parameterizations that match our style guide.
   # implicit_assignment_linter(allow_lazy = TRUE, allow_scoped = TRUE),
-  # implicit_integer_linter(allow_colon = TRUE),
   # system_time_linter = undesirable_function_linter(c(
   #   system.time = "Only run timings in benchmark.Rraw"
   # )),
@@ -26,7 +26,8 @@ linters = c(dt_linters, all_linters(
   #   setwd = NULL
   # )),
   undesirable_operator_linter(),
-  # TODO(lintr#2441): Use upstream implementation.
+  # TODO(lintr#2765): Use upstream implementation.
+  # assignment_linter(operator = "="),
   assignment_linter = NULL,
   absolute_path_linter = NULL, # too many false positives
   # TODO(lintr#2442): Use this once x[ , j, by] is supported.
@@ -84,7 +85,9 @@ exclusions = c(local({
       infix_spaces_linter = Inf,
       undesirable_function_linter = Inf
     )),
-    exclusion_for_dir(c("vignettes", "vignettes/fr"), list(
+    exclusion_for_dir(c("vignettes", "vignettes/fr", "vignettes/ru"), list(
+      # assignment_linter = Inf,
+      implicit_integer_linter = Inf,
       quotes_linter = Inf,
       sample_int_linter = Inf
       # strings_as_factors_linter = Inf

.ci/atime/tests.R

@@ -231,7 +231,20 @@ test.list <- atime::atime_test_list(
     },
     expr = data.table:::melt(DT, measure.vars = measure.vars),
     Slow = "fd24a3105953f7785ea7414678ed8e04524e6955", # Parent of the merge commit (https://github.com/Rdatatable/data.table/commit/ed72e398df76a0fcfd134a4ad92356690e4210ea) of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
-    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue
+    Fast = "ed72e398df76a0fcfd134a4ad92356690e4210ea"), # Merge commit of the PR (https://github.com/Rdatatable/data.table/pull/5054) that fixes the issue  # Test case created directly using the atime code below (not adapted from any other benchmark), based on the issue/fix PR https://github.com/Rdatatable/data.table/pull/5054#issue-930603663 "melt should be more efficient when there are missing input columns."
+
+  # Test case created from @tdhock's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2327396833, in turn adapted from @philippechataignon's comment https://github.com/Rdatatable/data.table/pull/6393#issuecomment-2326714012
+  "fwrite refactored in #6393" = atime::atime_test(
+    setup = {
+      set.seed(1)
+      NC = 10L
+      L <- data.table(i=1:N)
+      L[, paste0("V", 1:NC) := replicate(NC, rnorm(N), simplify=FALSE)]
+      out.csv <- tempfile()
+    },
+    expr = data.table::fwrite(L, out.csv, compress="gzip"),
+    Before = "f339aa64c426a9cd7cf2fcb13d91fc4ed353cd31", # Parent of the first commit https://github.com/Rdatatable/data.table/commit/fcc10d73a20837d0f1ad3278ee9168473afa5ff1 in the PR https://github.com/Rdatatable/data.table/pull/6393/commits with major change to fwrite with gzip.
+    PR = "3630413ae493a5a61b06c50e80d166924d2ef89a"), # Close-to-last merge commit in the PR.
 
   tests=extra.test.list)
 # nolint end: undesirable_operator_linter.

.ci/linters/c/alloc_linter.R

@@ -4,8 +4,9 @@
 #   2. Check the next line for a check like 'if (!x || !y)'
 alloc_linter = function(c_obj) {
   lines = c_obj$lines
-  # Be a bit more precise to avoid mentions in comments
-  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(]}", lines)
+  # Be a bit more precise to avoid mentions in comments, and allow
+  #   malloc(0) to be used for convenience (e.g. #6757)
+  alloc_lines = grep(R"{=\s*([(]\w+\s*[*][)])?[mc]alloc[(][^0]}", lines)
   if (!length(alloc_lines)) return()
   # int *tmp=(int*)malloc(...); or just int tmp=malloc(...);
   alloc_keys = lines[alloc_lines] |>

.ci/linters/c/cocci_linter.R

@@ -0,0 +1,20 @@
+cocci_linter = if (!nzchar(Sys.which("spatch"))) function(...) {} else function(c_obj) {
+  bad <- FALSE
+  for (spfile in list.files(".ci/linters/cocci", full.names = TRUE)) {
+    # Coccinelle parser gets confused sometimes, so ignore stderr and the exit code
+    out = suppressWarnings(system2(
+      "spatch",
+      shQuote(c(
+        "--sp-file", spfile, c_obj$path, "--recursive-includes",
+        "-I", R.home("include"), "-I", "src"
+      )),
+      stdout = TRUE, stderr = FALSE
+    ))
+    if (length(out) > 0) {
+      cat(sprintf("In file '%s', Coccinelle patch '%s' recommends the following changes:\n", c_obj$path, spfile))
+      writeLines(out)
+      bad <- TRUE
+    }
+  }
+  if (bad) stop("Please apply the changes above or fix the linter")
+}

.ci/linters/cocci/redundant_cast_to_same_pointer_type.cocci

@@ -0,0 +1,6 @@
+@@
+type T;
+T* E;
+@@
+- (T*)
+  E

.ci/linters/md/heading_id_linter.R

@@ -0,0 +1,26 @@
+any_mismatch = FALSE
+
+# ensure that ids are limited to alphanumerics and dashes
+# (in particular, dots and underscores break the links)
+check_header_ids = function(md) {
+  # A bit surprisingly, some headings don't start with a letter.
+  # We're interested in those that set an id to link to, i.e., end with {#id}.
+  heading_captures = regmatches(md, regexec("^#+ \\S.*[{]#([^}]*)[}]$", md))
+  lines_with_id = which(lengths(heading_captures) > 0)
+  ids = vapply(heading_captures[lines_with_id], `[`, '', 2)
+  # ids must start with a letter and consist of alphanumerics or dashes.
+  good_ids = grepl('^[A-Za-z][A-Za-z0-9-]*$', ids)
+  for (line in lines_with_id[!good_ids]) cat(sprintf(
+    "On line %d, bad heading id '%s':\n%s\n",
+    line, heading_captures[[line]][2], heading_captures[[line]][1]
+  ))
+  !all(good_ids)
+}
+
+any_error = FALSE
+for (vignette in list.files('vignettes', pattern = "[.]Rmd$", recursive = TRUE, full.name = TRUE)) {
+  cat(sprintf("Checking vignette file %s...\n", vignette))
+  rmd_lines = readLines(vignette)
+  any_error = check_header_ids(rmd_lines) || any_error
+}
+if (any_error) stop("Please fix the vignette issues above.")

.dev/CRAN_Release.cmd

@@ -3,22 +3,22 @@
 ###############################################
 
 # 1) Update messages for new release
-## (a) Update C template file: src/data.table.pot
-##     ideally, we are including _() wrapping in
-##     new PRs throughout dev cycle, and this step
-##     becomes about tying up loose ends
-## Check the output here for translatable messages
-xgettext -o /dev/stdout ./*.c \
-  --keyword=Rprintf --keyword=error --keyword=warning --keyword=STOP --keyword=DTWARN --keyword=Error --keyword=DTPRINT --keyword=snprintf:3
-
-## (b) Update R template file: src/R-data.table.pot
-##  NB: this relies on R >= 4.0 to remove a bug in update_pkg_po
-Rscript -e "tools::update_pkg_po('.')"
+dt_custom_translators = list(
+  R = 'catf:fmt|1',
+  # TODO(MichaelChirico/potools#318): restore snprintf:3 here too
+  src = c('STOP:1', 'DTWARN:1', 'DTPRINT:1')
+)
+message_db =
+  potools::get_message_data(custom_translation_functions = dt_custom_translators)
+potools::check_cracked_messages(message_db)
+potools::check_untranslated_cat(message_db)
+potools::check_untranslated_src(message_db)
+
+## (b) Update R template files (po/*.pot)
+potools::po_extract(custom_translation_functions = dt_custom_translators)
 
 # 2) Open a PR with the new templates & contact the translators
-#   * zh_CN: @hongyuanjia
-#   * pt_BR: @rffontenelle
-#   * es: @rikivillalba
+#    using @Rdatatable/<lang>, e.g. @Rdatatable/chinese
 ## Translators to submit commits with translations to this PR
 ##   [or perhaps, if we get several languages, each to open
 ##    its own PR and merge to main translation PR]
@@ -114,12 +114,6 @@ grep -P "\t" ./src/*.c
 grep -n "[^A-Za-z0-9]T[^A-Za-z0-9]" ./inst/tests/tests.Rraw
 grep -n "[^A-Za-z0-9]F[^A-Za-z0-9]" ./inst/tests/tests.Rraw
 
-# All integers internally should have L suffix to avoid lots of one-item coercions
-# Where 0 numeric is intended we should perhaps use 0.0 for clarity and make the grep easier
-# 1) tolerance=0 usages in setops.R are valid numeric 0, as are anything in strings
-# 2) leave the rollends default using roll>=0 though; comments in PR #3803
-grep -Enr "^[^#]*(?:\[|==|>|<|>=|<=|,|\(|\+)\s*[-]?[0-9]+[^0-9L:.e]" R | grep -Ev "stop|warning|tolerance"
-
 # Never use ifelse. fifelse for vectors when necessary (nothing yet)
 grep -Enr "\bifelse" R
 
@@ -135,10 +129,6 @@ grep -Fn "tryCatch" ./inst/tests/*.Rraw
 # All % in *.Rd should be escaped otherwise text gets silently chopped
 grep -n "[^\]%" ./man/*.Rd
 
-# if (a & b) is either invalid or inefficient (ditto for replace & with |);
-#   if(any(a [&|] b)) is appropriate b/c of collapsing the logical vector to scalar
-grep -nr "^[^#]*if[^&#]*[^&#\"][&][^&]" R | grep -Ev "if\s*[(](?:any|all)"
-
 # seal leak potential where two unprotected API calls are passed to the same
 # function call, usually involving install() or mkChar()
 # Greppable thanks to single lines and wide screens

.devcontainer/r-devel-alpine/Dockerfile

@@ -0,0 +1,22 @@
+FROM docker.io/rhub/r-minimal:devel
+
+RUN apk update \
+  && apk add --no-cache \
+     gcc git musl-dev openmp pkgconf tzdata zlib-dev \
+  && echo 'options("repos"="https://cloud.r-project.org")' >> /usr/local/lib/R/etc/Rprofile.site
+
+ENV TZDIR=/usr/share/zoneinfo
+
+COPY DESCRIPTION .
+
+RUN Rscript -e '                                          \ 
+read.dcf("DESCRIPTION", c("Imports", "Suggests")) |>      \
+  tools:::.split_dependencies() |>                        \
+  names() |>                                              \
+  setdiff(tools:::.get_standard_package_names()$base) |>  \
+  install.packages(repos="https://cloud.r-project.org")   \
+'
+
+# setup cc()
+WORKDIR /root
+COPY .devcontainer/.Rprofile .

.devcontainer/r-devel-alpine/devcontainer.json

@@ -0,0 +1,9 @@
+{
+    "build": { "dockerfile": "Dockerfile", "context": "../.." },
+    "customizations": { "vscode": {
+       "extensions": [
+        "REditorSupport.r",
+        "ms-vscode.cpptools-extension-pack"
+       ]
+    }}
+  }

.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,35 @@
+As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project members with the Committer role have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project members commit themselves to fairly and consistently apply these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team.
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community.
+
+
+## Reporting
+
+Project members with the Committer role or the CRAN Maintainer role are pledged to promptly address any reported issues. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to any individual with this role.
+
+Those who prefer to report in a way that is independent of the current Committers and Maintainer may instead contact the Community Engagement Coordinator by e-mailing [r.data.table\@gmail.com](mailto:[email protected]). Messages sent to this e-mail address will be visible only to the current Community Engagement Coordinator, a position always held by an individual who is not a Committer or CRAN Maintainer of the package.
+
+The current Committers are Toby Dylan Hocking (@tdhock), Matt Dowle (@mattdowle), Arun Srinivasan (@arunsrinivasan), Jan Gorecki (@jangorecki), Michael Chirico (@MichaelChirico), and Benjamin Schwendinger (@ben-schwen).
+
+The current CRAN Maintainer is Tyson Barrett (@tysonstanley).
+
+The current Community Engagement Coordinator is Kelly Bodwin (@kbodwin).
+
+All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. Complaint respondents are obligated to maintain confidentiality with regard to the reporter of an incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant, version 1.3.0](https://www.contributor-covenant.org/version/1/3/0/code-of-conduct/), available at [https://www.contributor-covenant.org/version/1/3/0/](https://www.contributor-covenant.org/version/1/3/0/), and the Swift Code of Conduct.